Thread 140 (Thread 0x7f4d039ff700 (LWP 3419433) "IOThreadPool9"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d4301f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d4301f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d4301f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d4301f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d4301f0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 139 (Thread 0x7f4d047ff700 (LWP 3419432) "IOThreadPool8"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d4cf1e0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d4cf1e0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d4cf1e0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d4cf1e0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d4cf1e0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 138 (Thread 0x7f4d055ff700 (LWP 3419431) "IOThreadPool7"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d0d073630) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d0d073630) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d0d073630) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d0d073630) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d0d073620) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 137 (Thread 0x7f4d063ff700 (LWP 3419430) "IOThreadPool6"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d47a1f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d47a1f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d47a1f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d47a1f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d47a1f0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 136 (Thread 0x7f4d091ff700 (LWP 3419429) "IOThreadPool5"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d4c41f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d4c41f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d4c41f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d4c41f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d4c41f0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 135 (Thread 0x7f4d09fff700 (LWP 3419428) "IOThreadPool4"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d4661f240) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d4661f240) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d4661f240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d4661f240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d4661f230) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 134 (Thread 0x7f4d0adff700 (LWP 3419426) "IOThreadPool3"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d49b1e0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d49b1e0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d49b1e0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d49b1e0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d49b1e0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 133 (Thread 0x7f4d089f4700 (LWP 3419416) "IOThreadPool2"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d4661f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d4661f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d4661f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d4661f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d4661f0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 132 (Thread 0x7f4d081f3700 (LWP 3419413) "IOThreadPool1"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d0d073550) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d0d073550) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d0d073550) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d0d073550) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d0d073540) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 131 (Thread 0x7f4d079f2700 (LWP 3419400) "IOThreadPool0"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d49e1f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d49e1f0f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d49e1f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d49e1f0f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d49e1f0e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 130 (Thread 0x7f4d071f1700 (LWP 3415271) "hazptr-tpe-0"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a71508 in folly::detail::futexWaitImpl const, std::chrono::time_point > > > (futex=0x7f4d4dadd3d0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:81 #3 0x0000000002a6cf6e in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadd3d0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #4 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #5 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadd3d0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #6 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadd3d0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #7 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d42aff4c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #8 0x0000000004643b20 in folly::UnboundedBlockingQueue::try_take_for(std::chrono::duration >) () #9 0x000000000463d967 in folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr) () #10 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #11 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d42b5d590) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #12 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d42b5d590) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #13 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #14 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #15 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d42b5d590) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #16 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d42b5d590) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #17 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d42b5d580) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #18 0x0000000004b4043f in execute_native_thread_routine () #19 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #20 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 129 (Thread 0x7f4d0efff700 (LWP 3413841) "Acceptor Thread"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d1d842200) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d1d842200) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d1d842200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d1d842200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d1d8421f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 128 (Thread 0x7f4d0fdfd700 (LWP 3413840) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb780, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb780, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb780) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e9c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 127 (Thread 0x7f4d0fefe700 (LWP 3413839) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb798, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb798, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb798) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e9c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 126 (Thread 0x7f4d101ff700 (LWP 3413838) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb768, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb768, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb768) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 125 (Thread 0x7f4d171fe700 (LWP 3413837) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb750, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb750, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb750) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 124 (Thread 0x7f4d183f0700 (LWP 3413836) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb738, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb738, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb738) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 123 (Thread 0x7f4d1ddfe700 (LWP 3413835) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb720, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb720, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb720) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 122 (Thread 0x7f4d1ebfe700 (LWP 3413834) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb708, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb708, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb708) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 121 (Thread 0x7f4d209fe700 (LWP 3413833) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb6f0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb6f0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb6f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 120 (Thread 0x7f4d237fe700 (LWP 3413832) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb6d8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb6d8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb6d8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 119 (Thread 0x7f4d245fe700 (LWP 3413831) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb6a8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb6a8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb6a8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 118 (Thread 0x7f4d253fe700 (LWP 3413830) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb6c0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb6c0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb6c0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 117 (Thread 0x7f4d261fe700 (LWP 3413829) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb690, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb690, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb690) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 116 (Thread 0x7f4d26ffe700 (LWP 3413828) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb678, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb678, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb678) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 115 (Thread 0x7f4d27dfe700 (LWP 3413827) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb660, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb660, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb660) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 114 (Thread 0x7f4d293fe700 (LWP 3413826) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb648, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb648, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb648) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 113 (Thread 0x7f4d2a1fc700 (LWP 3413825) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb630, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb630, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb630) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 112 (Thread 0x7f4d2bffe700 (LWP 3413824) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb618, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb618, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb618) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 111 (Thread 0x7f4d2cdfe700 (LWP 3413823) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb600, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb600, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb600) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 110 (Thread 0x7f4d2dbfe700 (LWP 3413822) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb5e8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb5e8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb5e8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 109 (Thread 0x7f4d2e9fe700 (LWP 3413821) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb5a0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb5a0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb5a0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 108 (Thread 0x7f4d301fe700 (LWP 3413820) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb5b8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb5b8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb5b8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 107 (Thread 0x7f4d30ffe700 (LWP 3413819) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb5d0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb5d0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb5d0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 106 (Thread 0x7f4d31dfe700 (LWP 3413818) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb588, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb588, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb588) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 105 (Thread 0x7f4d32bfe700 (LWP 3413817) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb570, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb570, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb570) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 104 (Thread 0x7f4d339fe700 (LWP 3413816) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb540, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb540, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb540) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 103 (Thread 0x7f4d34bfe700 (LWP 3413815) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb558, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb558, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb558) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 102 (Thread 0x7f4d359fd700 (LWP 3413814) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb528, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb528, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb528) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 101 (Thread 0x7f4d373fd700 (LWP 3413813) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb510, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb510, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb510) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 100 (Thread 0x7f4d389fe700 (LWP 3413812) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb4f8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb4f8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb4f8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 99 (Thread 0x7f4d17bff700 (LWP 3413811) "Acceptor Thread"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d1d040240) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d1d040240) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d1d040240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d1d040240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d1d040230) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 98 (Thread 0x7f4d397fe700 (LWP 3413810) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb4e0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb4e0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb4e0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 97 (Thread 0x7f4d1adff700 (LWP 3413808) "Acceptor Thread"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d1d440240) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d1d440240) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d1d440240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d1d440240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d1d440230) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 96 (Thread 0x7f4d3a5fe700 (LWP 3413809) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb4c8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb4c8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb4c8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 95 (Thread 0x7f4d3b3fe700 (LWP 3413807) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb4b0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb4b0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb4b0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 94 (Thread 0x7f4d3c1fe700 (LWP 3413806) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb498, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb498, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb498) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 93 (Thread 0x7f4d3cffe700 (LWP 3413805) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb480, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb480, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb480) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80e3c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 92 (Thread 0x7f4d3ddfe700 (LWP 3413804) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb468, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb468, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb468) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80ddc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 91 (Thread 0x7f4d3ebfe700 (LWP 3413803) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb450, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb450, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb450) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80ddc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 90 (Thread 0x7f4d3f9fe700 (LWP 3413802) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb438, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb438, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb438) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80d7c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 89 (Thread 0x7f4d407fe700 (LWP 3413801) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb420, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb420, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb420) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80d7c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 88 (Thread 0x7f4d415fe700 (LWP 3413800) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb408, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb408, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb408) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80d1c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 87 (Thread 0x7f4d427fd700 (LWP 3413799) "reader-pool-pri"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb3f0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb3f0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb3f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d1d80d1c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 86 (Thread 0x7f4d1e7ff700 (LWP 3413798) "nebula-storaged"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x000000000423f755 in apache::thrift::ThriftServer::serve() () #7 0x0000000002a57e57 in nebula::storage::StorageServer::::operator()(void) const (__closure=0x7f4d52862e98) at /root/src/wwl/nebula/src/storage/StorageServer.cpp:288 #8 0x0000000002a67256 in std::__invoke_impl >(std::__invoke_other, nebula::storage::StorageServer:: &&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #9 0x0000000002a5fb6e in std::__invoke >(nebula::storage::StorageServer:: &&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #10 0x0000000002aa523e in std::thread::_Invoker > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x7f4d52862e98) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #11 0x0000000002aa4745 in std::thread::_Invoker > >::operator()(void) (this=0x7f4d52862e98) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #12 0x0000000002aa19f4 in std::thread::_State_impl > > >::_M_run(void) (this=0x7f4d52862e90) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #13 0x0000000004b4043f in execute_native_thread_routine () #14 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 85 (Thread 0x7f4d1f3ff700 (LWP 3413797) "nebula-storaged"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x000000000423f755 in apache::thrift::ThriftServer::serve() () #7 0x0000000002a5772d in nebula::storage::StorageServer::::operator()(void) const (__closure=0x7f4d52862e78) at /root/src/wwl/nebula/src/storage/StorageServer.cpp:263 #8 0x0000000002a67016 in std::__invoke_impl >(std::__invoke_other, nebula::storage::StorageServer:: &&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #9 0x0000000002a5f9e8 in std::__invoke >(nebula::storage::StorageServer:: &&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #10 0x0000000002aa526a in std::thread::_Invoker > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x7f4d52862e78) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #11 0x0000000002aa476b in std::thread::_Invoker > >::operator()(void) (this=0x7f4d52862e78) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #12 0x0000000002aa1a14 in std::thread::_State_impl > > >::_M_run(void) (this=0x7f4d52862e70) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #13 0x0000000004b4043f in execute_native_thread_routine () #14 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 84 (Thread 0x7f4d1a3f4700 (LWP 3413796) "nebula-storaged"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x000000000423f755 in apache::thrift::ThriftServer::serve() () #7 0x0000000002a56fb5 in nebula::storage::StorageServer::::operator()(void) const (__closure=0x7f4d52862e88) at /root/src/wwl/nebula/src/storage/StorageServer.cpp:234 #8 0x0000000002a66d76 in std::__invoke_impl >(std::__invoke_other, nebula::storage::StorageServer:: &&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #9 0x0000000002a5f80a in std::__invoke >(nebula::storage::StorageServer:: &&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #10 0x0000000002aa5296 in std::thread::_Invoker > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x7f4d52862e88) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #11 0x0000000002aa4791 in std::thread::_Invoker > >::operator()(void) (this=0x7f4d52862e88) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #12 0x0000000002aa1a34 in std::thread::_State_impl > > >::_M_run(void) (this=0x7f4d52862e80) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #13 0x0000000004b4043f in execute_native_thread_routine () #14 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 83 (Thread 0x7f4d19bf3700 (LWP 3413795) "nebula-storaged"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x0000000002bd6286 in nebula::storage::AdminTaskManager::::operator()(void) const (__closure=0x7f4d52862e68) at /root/src/wwl/nebula/src/storage/admin/AdminTaskManager.cpp:48 #3 0x0000000002bdef94 in std::__invoke_impl >(std::__invoke_other, nebula::storage::AdminTaskManager:: &&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #4 0x0000000002bdb276 in std::__invoke >(nebula::storage::AdminTaskManager:: &&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #5 0x0000000002bf836c in std::thread::_Invoker > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x7f4d52862e68) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #6 0x0000000002bf8325 in std::thread::_Invoker > >::operator()(void) (this=0x7f4d52862e68) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #7 0x0000000002bf8166 in std::thread::_State_impl > > >::_M_run(void) (this=0x7f4d52862e60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #8 0x0000000004b4043f in execute_native_thread_routine () #9 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 82 (Thread 0x7f4d193f2700 (LWP 3413794) "nebula-storaged"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a71508 in folly::detail::futexWaitImpl const, std::chrono::time_point > > > (futex=0x7f4d4dadbf18, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:81 #3 0x0000000002a6cf6e in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadbf18, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #4 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #5 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadbf18, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #6 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadbf18) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #7 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x5d8bd40 , deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #8 0x0000000002aa4d43 in folly::detail::LifoSemBase, std::atomic>::try_wait_for > (this=0x5d8bd40 , timeout=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:489 #9 0x0000000002aa3a1b in folly::UnboundedBlockingQueue >::try_take_for (this=0x5d8bd00 , time=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #10 0x0000000002bd85ca in nebula::storage::AdminTaskManager::schedule (this=0x5d8b480 ) at /root/src/wwl/nebula/src/storage/admin/AdminTaskManager.cpp:235 #11 0x0000000002bf7b1e in std::__invoke_impl (__f=@0x7f4d528bde60: (void (nebula::storage::AdminTaskManager::*)(nebula::storage::AdminTaskManager * const)) 0x2bd848c , __t=@0x7f4d528bde70: 0x5d8b480 ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #12 0x0000000002bf79e3 in std::__invoke (__fn=@0x7f4d528bde60: (void (nebula::storage::AdminTaskManager::*)(nebula::storage::AdminTaskManager * const)) 0x2bd848c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000002bf76c4 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d528bde60, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #14 0x0000000002bf66b7 in std::_Bind::operator()<, void>() (this=0x7f4d528bde60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #15 0x0000000002bf50a7 in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #16 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d52858670) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #17 0x0000000002bda7d7 in nebula::thread::GenericWorker::addTask(void (nebula::storage::AdminTaskManager::*&&)(), nebula::storage::AdminTaskManager*&&)::{lambda()#1}::operator()() const (__closure=0x7f4d22f4c9c0) at /root/src/wwl/nebula/src/common/thread/GenericWorker.h:180 #18 0x0000000002bed0ca in std::_Function_handler(void (nebula::storage::AdminTaskManager::*&&)(), nebula::storage::AdminTaskManager*&&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #19 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d22f4c8a0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #20 0x0000000003bbdfd8 in nebula::thread::GenericWorker::onNotify (this=0x7f4d52842f00) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:114 #21 0x0000000003bbd6b2 in nebula::thread::GenericWorker::::operator()(int, int16_t, void *) const (__closure=0x0, fd=189, arg=0x7f4d52842f00) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:56 #22 0x0000000003bbd700 in nebula::thread::GenericWorker::::_FUN(int, int16_t, void *) () at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:57 #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842f00) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #26 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d1f400000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d1f400010: 0x7f4d52842f00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #27 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d1f400000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #28 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d1f400000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #29 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d1f400000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #30 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #31 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d193efb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #32 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #33 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286ccf0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #34 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286ccf0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #35 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286ccb8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #36 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286ccb8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #37 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286ccb0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #38 0x0000000004b4043f in execute_native_thread_routine () #39 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #40 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 81 (Thread 0x7f4d18bf1700 (LWP 3413793) "nebula-storaged"): #0 0x00007f4d531c0de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000040efc72 in rocksdb::port::CondVar::TimedWait(unsigned long) () #2 0x0000000003f47e7b in rocksdb::Timer::Run() () #3 0x0000000004b4043f in execute_native_thread_routine () #4 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 80 (Thread 0x7f4d213ff700 (LWP 3413777) "rocksdb:high0"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x00000000040bcbea in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #3 0x00000000040bcf65 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #4 0x0000000004b4043f in execute_native_thread_routine () #5 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 79 (Thread 0x7f4d21efd700 (LWP 3413776) "rocksdb:low2"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x00000000040bcbea in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #3 0x00000000040bcf65 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #4 0x0000000004b4043f in execute_native_thread_routine () #5 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 78 (Thread 0x7f4d226fe700 (LWP 3413775) "rocksdb:low1"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x00000000040bcbea in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #3 0x00000000040bcf65 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #4 0x0000000004b4043f in execute_native_thread_routine () #5 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 77 (Thread 0x7f4d22eff700 (LWP 3413774) "rocksdb:low0"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x00000000040bcbea in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #3 0x00000000040bcf65 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #4 0x0000000004b4043f in execute_native_thread_routine () #5 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 76 (Thread 0x7f4d23fff700 (LWP 3413773) "nebula-storaged"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x000000000459e4d1 in folly::fibers::Baton::waitThread() () #3 0x000000000459e7c8 in folly::fibers::Baton::wait() () #4 0x0000000002c899b1 in folly::futures::detail::waitImpl, folly::Unit> (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2090 #5 0x0000000002c8906c in folly::SemiFuture::wait() & (this=0x7f4d23ffc288) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2197 #6 0x0000000002c881da in folly::SemiFuture::getTry() && (this=0x7f4d23ffc288) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2254 #7 0x0000000002c872a4 in folly::Future::getTry() && (this=0x7f4d23ffc568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2333 #8 0x0000000002c85830 in folly::Future::get() && (this=0x7f4d23ffc568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2323 #9 0x0000000002c851fc in nebula::storage::ChainResumeProcessor::process (this=0x7f4d23ffc618) at /root/src/wwl/nebula/src/storage/transaction/ChainResumeProcessor.cpp:58 #10 0x0000000002c4c2f6 in nebula::storage::TransactionManager::resumeThread (this=0x7f4d48abd000) at /root/src/wwl/nebula/src/storage/transaction/TransactionManager.cpp:88 #11 0x0000000002c6f873 in std::__invoke_impl (__f=@0x7f4d2328b200: (void (nebula::storage::TransactionManager::*)(nebula::storage::TransactionManager * const)) 0x2c4c2a6 , __t=@0x7f4d2328b210: 0x7f4d48abd000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #12 0x0000000002c6f7e3 in std::__invoke (__fn=@0x7f4d2328b200: (void (nebula::storage::TransactionManager::*)(nebula::storage::TransactionManager * const)) 0x2c4c2a6 ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000002c6f71e in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d2328b200, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #14 0x0000000002c6f52f in std::_Bind::operator()<, void>() (this=0x7f4d2328b200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #15 0x0000000002c6e349 in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #16 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d23204070) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #17 0x0000000002c4fc7b in nebula::thread::GenericWorker::addDelayTask(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1}::operator()() const (__closure=0x7f4d23204030) at /root/src/wwl/nebula/src/common/thread/GenericWorker.h:216 #18 0x0000000002c6aace in std::__invoke_impl(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1}&>(std::__invoke_other, nebula::thread::GenericWorker::addDelayTask(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1}&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #19 0x0000000002c67c1a in std::__invoke(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1}&>(void (nebula::storage::TransactionManager::*&&)()) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #20 0x0000000002c64a2e in std::_Bind(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1} ()>::__call(std::tuple<>&&, std::_Index_tuple<>) (this=0x7f4d23204030, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #21 0x0000000002c6207b in std::_Bind(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1} ()>::operator()<, void>() (this=0x7f4d23204030) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #22 0x0000000002c5df9d in std::_Function_handler(unsigned long, void (nebula::storage::TransactionManager::*&&)(), nebula::storage::TransactionManager*&&)::{lambda()#1} ()> >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #23 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d2320d068) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #24 0x0000000003bbde20 in nebula::thread::GenericWorker::::operator()(int, int16_t, void *) const (__closure=0x0, fd=-1, arg=0x7f4d2320d050) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:127 #25 0x0000000003bbdeaa in nebula::thread::GenericWorker::::_FUN(int, int16_t, void *) () at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:131 #26 0x00000000046c284f in ?? () #27 0x00000000046c316f in event_base_loop () #28 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842700) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #29 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d23200000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d23200010: 0x7f4d52842700) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #30 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d23200000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #31 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d23200000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #32 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d23200000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #33 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #34 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d23ffcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #35 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #36 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c890: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #37 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c890: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #38 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c858) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #39 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c858) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #40 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c850) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #41 0x0000000004b4043f in execute_native_thread_routine () #42 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #43 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 75 (Thread 0x7f4d24dff700 (LWP 3413772) "IOThreadPool0"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d28a20240) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d28a20240) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d28a20240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d28a20240) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d28a20230) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 74 (Thread 0x7f4d25bff700 (LWP 3413771) "HTTPSrvExec3"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d28a20160) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d28a20160) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d28a20160) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d28a20160) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d28a20150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 73 (Thread 0x7f4d269ff700 (LWP 3413770) "HTTPSrvExec2"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d28a200f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d28a200f0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d28a200f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d28a200f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d28a200e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 72 (Thread 0x7f4d277ff700 (LWP 3413769) "HTTPSrvExec1"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d28a20080) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d28a20080) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d28a20080) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d28a20080) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d28a20070) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 71 (Thread 0x7f4d285ff700 (LWP 3413768) "HTTPSrvExec0"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d28a20010) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d28a20010) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d28a20010) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d28a20010) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d28a20000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 70 (Thread 0x7f4d29bff700 (LWP 3413767) "webservice-list"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x00000000044654fa in proxygen::HTTPServer::start(std::function, std::function) () #7 0x0000000003b77dd8 in nebula::WebService::::operator()(void) const (__closure=0x7f4d28a00000) at /root/src/wwl/nebula/src/webservice/WebService.cpp:115 #8 0x0000000003b80706 in std::__invoke_impl&>(std::__invoke_other, nebula::WebService:: &) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #9 0x0000000003b80603 in std::__invoke&>(nebula::WebService:: &) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #10 0x0000000003b804c8 in std::_Bind()>::__call(std::tuple<> &&, std::_Index_tuple<>) (this=0x7f4d28a00000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #11 0x0000000003b80189 in std::_Bind()>::operator()<>(void) (this=0x7f4d28a00000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #12 0x0000000003b7fb92 in std::_Function_handler()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #13 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d29bfcb50) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #14 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #15 0x0000000003b7e816 in std::__invoke_impl&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> >(std::__invoke_other, void (*&&)(const std::__cxx11::basic_string, std::allocator > &, const std::function &)) (__f=@0x7f4d52809b98: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #16 0x0000000003b7d3f6 in std::__invoke&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> >(void (*&&)(const std::__cxx11::basic_string, std::allocator > &, const std::function &)) (__fn=@0x7f4d52809b98: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #17 0x0000000003b809e4 in std::thread::_Invoker, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > >::_M_invoke<0, 1, 2>(std::_Index_tuple<0, 1, 2>) (this=0x7f4d52809b48) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #18 0x0000000003b8097b in std::thread::_Invoker, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > >::operator()(void) (this=0x7f4d52809b48) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #19 0x0000000003b8095a in std::thread::_State_impl, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > > >::_M_run(void) (this=0x7f4d52809b40) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #20 0x0000000004b4043f in execute_native_thread_routine () #21 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 69 (Thread 0x7f4d2a9fd700 (LWP 3413766) "http thread poo"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842600) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d29c00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d29c00010: 0x7f4d52842600) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d29c00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d29c00000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d29c00000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d2a9fab60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c7f0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c7f0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c7b8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c7b8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c7b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 68 (Thread 0x7f4d2b1fe700 (LWP 3413765) "http thread poo"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842500) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d28600000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d28600010: 0x7f4d52842500) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d28600000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d28600000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d28600000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d2b1fbb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c7a0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c7a0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c768) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c768) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c760) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 67 (Thread 0x7f4d2b9ff700 (LWP 3413764) "http thread poo"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842300) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d28e00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d28e00010: 0x7f4d52842300) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d28e00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d28e00000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d28e00000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d2b9fcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c750: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c750: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c718) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c718) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c710) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 66 (Thread 0x7f4d2c7ff700 (LWP 3413763) "Acceptor Thread"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #7 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #8 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d334105c0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #9 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d334105c0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #10 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d334105c0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d334105c0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d334105b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 65 (Thread 0x7f4d2d5ff700 (LWP 3413762) "IOThreadPool15"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d2d5fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d2327b3c0, req=..., serializedRequest=..., ctx=0x7f4d2c8aad10, eb=0x7f4d2c804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d2327b3c0, req=..., serializedRequest=..., ctx=0x7f4d2c8aad10, eb=0x7f4d2c804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d2327b3c0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d2c8aad10, eb=0x7f4d2c804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d2327b3c0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d2c8aad10, eb=0x7f4d2c804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d2327b3c0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d2c8aad10, eb=0x7f4d2c804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d33410550) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d33410550) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d33410550) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d33410550) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d33410540) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 64 (Thread 0x7f4d2e3ff700 (LWP 3413761) "IOThreadPool14"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d2e3fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d02d2e980, req=..., serializedRequest=..., ctx=0x7f4d2d67d710, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d02d2e980, req=..., serializedRequest=..., ctx=0x7f4d2d67d710, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d02d2e980, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d2d67d710, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d02d2e980, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d2d67d710, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d02d2e980, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d2d67d710, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d334104e0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d334104e0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d334104e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d334104e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d334104d0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 63 (Thread 0x7f4d2f1ff700 (LWP 3413760) "IOThreadPool13"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d2f1fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cf104ad80, req=..., serializedRequest=..., ctx=0x7f4d2e47ef10, eb=0x7f4d2e404000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cf104ad80, req=..., serializedRequest=..., ctx=0x7f4d2e47ef10, eb=0x7f4d2e404000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cf104ad80, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d2e47ef10, eb=0x7f4d2e404000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cf104ad80, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d2e47ef10, eb=0x7f4d2e404000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cf104ad80, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d2e47ef10, eb=0x7f4d2e404000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d33410470) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d33410470) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d33410470) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d33410470) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d33410460) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 62 (Thread 0x7f4d309ff700 (LWP 3413759) "IOThreadPool12"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d309fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d0491bac0, req=..., serializedRequest=..., ctx=0x7f4d2fcba410, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d0491bac0, req=..., serializedRequest=..., ctx=0x7f4d2fcba410, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d0491bac0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d2fcba410, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d0491bac0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d2fcba410, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d0491bac0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d2fcba410, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d33410400) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d33410400) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d33410400) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d33410400) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d334103f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 61 (Thread 0x7f4d317ff700 (LWP 3413758) "IOThreadPool11"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d317fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d0491bae0, req=..., serializedRequest=..., ctx=0x7f4d30ad0510, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d0491bae0, req=..., serializedRequest=..., ctx=0x7f4d30ad0510, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d0491bae0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d30ad0510, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d0491bae0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d30ad0510, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d0491bae0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d30ad0510, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d33410390) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d33410390) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d33410390) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d33410390) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d33410380) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 60 (Thread 0x7f4d325ff700 (LWP 3413757) "IOThreadPool10"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d325fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d0655e7e0, req=..., serializedRequest=..., ctx=0x7f4d31922e10, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d0655e7e0, req=..., serializedRequest=..., ctx=0x7f4d31922e10, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d0655e7e0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d31922e10, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d0655e7e0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d31922e10, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d0655e7e0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d31922e10, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d33410320) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d33410320) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d33410320) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d33410320) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d33410310) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 59 (Thread 0x7f4d333ff700 (LWP 3413756) "IOThreadPool9"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d333fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d23285980, req=..., serializedRequest=..., ctx=0x7f4d32640510, eb=0x7f4d32604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d23285980, req=..., serializedRequest=..., ctx=0x7f4d32640510, eb=0x7f4d32604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d23285980, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d32640510, eb=0x7f4d32604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d23285980, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d32640510, eb=0x7f4d32604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d23285980, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d32640510, eb=0x7f4d32604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d334102b0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d334102b0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d334102b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d334102b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d334102a0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 58 (Thread 0x7f4d341ff700 (LWP 3413755) "nebula-storaged"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #4 0x00000000045ecca6 in folly::EventBase::loop() () #5 0x00000000045ee756 in folly::EventBase::loopForever() () #6 0x0000000003d9496f in nebula::raftex::RaftexService::serve (this=0x7f4d528323b0) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:105 #7 0x0000000003d940f7 in nebula::raftex::RaftexService::::operator()(void) const (__closure=0x7f4d52862c78) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:41 #8 0x0000000003d95eb6 in std::__invoke_impl >(std::__invoke_other, nebula::raftex::RaftexService:: &&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #9 0x0000000003d957cc in std::__invoke >(nebula::raftex::RaftexService:: &&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #10 0x0000000003d97d76 in std::thread::_Invoker > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x7f4d52862c78) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #11 0x0000000003d97d45 in std::thread::_Invoker > >::operator()(void) (this=0x7f4d52862c78) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #12 0x0000000003d97d24 in std::thread::_State_impl > > >::_M_run(void) (this=0x7f4d52862c70) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #13 0x0000000004b4043f in execute_native_thread_routine () #14 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 57 (Thread 0x7f4d355ff700 (LWP 3413754) "nebula-storaged"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d5282ca10) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d34200000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d34200010: 0x7f4d5282ca10) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d34200000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d34200000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d34200000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d355fcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c4d0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c4d0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c498) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c498) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c490) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 56 (Thread 0x7f4d361fe700 (LWP 3413753) "nebula-bgworker"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842200) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d34600000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d34600010: 0x7f4d52842200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d34600000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d34600000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d34600000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d361fbb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c480: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c480: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c448) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c448) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c440) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 55 (Thread 0x7f4d369ff700 (LWP 3413752) "nebula-bgworker"): #0 0x00007f4d531c0a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000004b32e1c in std::condition_variable::wait(std::unique_lock&) () #2 0x0000000003d590e4 in std::condition_variable::wait(std::unique_lock&, nebula::raftex::Host::reset()::{lambda()#1}) (this=0x7f4d3843d3b0, __lock=..., __p=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/condition_variable:99 #3 0x0000000003d448a5 in nebula::raftex::Host::reset (this=0x7f4d3843d310) at /root/src/wwl/nebula/src/kvstore/raftex/Host.h:55 #4 0x0000000003d509f9 in nebula::raftex::RaftPart::leaderElection (this=0x7f4d384be010) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1144 #5 0x0000000003d51094 in nebula::raftex::RaftPart::statusPolling (this=0x7f4d384be010, startTime=1634630400469) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1188 #6 0x0000000003d50fcb in nebula::raftex::RaftPart::::operator()(void) const (__closure=0x7f4cf1843220) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1209 #7 0x0000000003d922ca in std::__invoke_impl&>(std::__invoke_other, nebula::raftex::RaftPart:: &) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #8 0x0000000003d920e7 in std::__invoke&>(nebula::raftex::RaftPart:: &) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #9 0x0000000003d918f6 in std::_Bind()>::__call(std::tuple<> &&, std::_Index_tuple<>) (this=0x7f4cf1843220, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #10 0x0000000003d910ab in std::_Bind()>::operator()<>(void) (this=0x7f4cf1843220) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #11 0x0000000003d90402 in std::_Function_handler()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #12 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d46118010) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #13 0x0000000003d64d99 in nebula::thread::GenericWorker::::operator()(void) const (__closure=0x7f4d04931210) at /root/src/wwl/nebula/src/common/thread/GenericWorker.h:216 #14 0x0000000003d8ee5b in std::__invoke_impl; Args = {}]::&>(std::__invoke_other, nebula::thread::GenericWorker:: &) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #15 0x0000000003d8d6b2 in std::__invoke; Args = {}]::&>(nebula::thread::GenericWorker:: &) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #16 0x0000000003d8b600 in std::_Bind; Args = {}; typename std::enable_if::type>::value, folly::SemiFuture >::type = folly::SemiFuture; size_t = long unsigned int]::()>::__call(std::tuple<> &&, std::_Index_tuple<>) (this=0x7f4d04931210, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #17 0x0000000003d881f3 in std::_Bind; Args = {}; typename std::enable_if::type>::value, folly::SemiFuture >::type = folly::SemiFuture; size_t = long unsigned int]::()>::operator()<>(void) (this=0x7f4d04931210) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #18 0x0000000003d83987 in std::_Function_handler; Args = {}; typename std::enable_if::type>::value, folly::SemiFuture >::type = folly::SemiFuture; size_t = long unsigned int]::()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #19 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d36e11178) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #20 0x0000000003bbde20 in nebula::thread::GenericWorker::::operator()(int, int16_t, void *) const (__closure=0x0, fd=-1, arg=0x7f4d36e11160) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:127 #21 0x0000000003bbdeaa in nebula::thread::GenericWorker::::_FUN(int, int16_t, void *) () at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:131 #22 0x00000000046c284f in ?? () #23 0x00000000046c316f in event_base_loop () #24 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842100) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #25 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d34bff000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d34bff010: 0x7f4d52842100) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #26 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d34bff000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #27 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d34bff000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #28 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d34bff000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #29 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #30 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d369fcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #31 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #32 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c430: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c430: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c3f8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c3f8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c3f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 54 (Thread 0x7f4d37bfe700 (LWP 3413751) "nebula-bgworker"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d52842000) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d36e00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d36e00010: 0x7f4d52842000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d36e00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d36e00000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d36e00000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d37bfbb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c3e0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c3e0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c3a8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c3a8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c3a0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 53 (Thread 0x7f4d383ff700 (LWP 3413750) "nebula-bgworker"): #0 0x00007f4d52ee5fd3 in epoll_wait () from /lib64/libc.so.6 #1 0x00000000046cc575 in ?? () #2 0x00000000046c2f74 in event_base_loop () #3 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d5285ef00) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #4 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d36a00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d36a00010: 0x7f4d5285ef00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #5 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d36a00000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #6 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d36a00000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #7 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d36a00000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #8 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #9 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d383fcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #10 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #11 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c390: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #12 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c390: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c358) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #14 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c358) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #15 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c350) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #16 0x0000000004b4043f in execute_native_thread_routine () #17 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 52 (Thread 0x7f4d391ff700 (LWP 3413749) "nebula-storaged"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x000000000459e4d1 in folly::fibers::Baton::waitThread() () #3 0x000000000459e7c8 in folly::fibers::Baton::wait() () #4 0x0000000002edeee7 in folly::futures::detail::waitImpl >, nebula::StatusOr > (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2090 #5 0x0000000002e87726 in folly::SemiFuture >::wait() & (this=0x7f4d391fbc48) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2197 #6 0x0000000002e42fd2 in folly::SemiFuture >::getTry() && (this=0x7f4d391fbc48) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2254 #7 0x0000000002df9926 in folly::Future >::getTry() && (this=0x7f4d391fc518) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2333 #8 0x0000000002d86a2e in folly::Future >::get() && (this=0x7f4d391fc518) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:2323 #9 0x0000000002d6103e in nebula::meta::MetaClient::loadData (this=0x7f4d52827e00) at /root/src/wwl/nebula/src/clients/meta/MetaClient.cpp:267 #10 0x0000000002d5f67e in nebula::meta::MetaClient::heartBeatThreadFunc (this=0x7f4d52827e00) at /root/src/wwl/nebula/src/clients/meta/MetaClient.cpp:146 #11 0x0000000002f9ac78 in std::__invoke_impl (__f=@0x7f4cf3587c00: (void (nebula::meta::MetaClient::*)(nebula::meta::MetaClient * const)) 0x2d5f512 , __t=@0x7f4cf3587c10: 0x7f4d52827e00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #12 0x0000000002f9945f in std::__invoke (__fn=@0x7f4cf3587c00: (void (nebula::meta::MetaClient::*)(nebula::meta::MetaClient * const)) 0x2d5f512 ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #13 0x0000000002f96424 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4cf3587c00, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #14 0x0000000002f90c7b in std::_Bind::operator()<, void>() (this=0x7f4cf3587c00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #15 0x0000000002f8ba85 in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #16 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d49019130) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #17 0x0000000002d855e1 in nebula::thread::GenericWorker::addDelayTask(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1}::operator()() const (__closure=0x7f4d392104b0) at /root/src/wwl/nebula/src/common/thread/GenericWorker.h:216 #18 0x0000000002f75172 in std::__invoke_impl(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1}&>(std::__invoke_other, nebula::thread::GenericWorker::addDelayTask(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1}&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #19 0x0000000002f63008 in std::__invoke(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1}&>(void (nebula::meta::MetaClient::*&&)()) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #20 0x0000000002f49f52 in std::_Bind(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1} ()>::__call(std::tuple<>&&, std::_Index_tuple<>) (this=0x7f4d392104b0, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #21 0x0000000002f189d1 in std::_Bind(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1} ()>::operator()<, void>() (this=0x7f4d392104b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #22 0x0000000002edd675 in std::_Function_handler(unsigned long, void (nebula::meta::MetaClient::*&&)(), nebula::meta::MetaClient*&&)::{lambda()#1} ()> >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #23 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d44108ab8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #24 0x0000000003bbde20 in nebula::thread::GenericWorker::::operator()(int, int16_t, void *) const (__closure=0x0, fd=-1, arg=0x7f4d44108aa0) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:127 #25 0x0000000003bbdeaa in nebula::thread::GenericWorker::::_FUN(int, int16_t, void *) () at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:131 #26 0x00000000046c284f in ?? () #27 0x00000000046c316f in event_base_loop () #28 0x0000000003bbdc56 in nebula::thread::GenericWorker::loop (this=0x7f4d5285eb00) at /root/src/wwl/nebula/src/common/thread/GenericWorker.cpp:90 #29 0x0000000003bc1c73 in std::__invoke_impl (__f=@0x7f4d38400000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c , __t=@0x7f4d38400010: 0x7f4d5285eb00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:73 #30 0x0000000003bc1bd8 in std::__invoke (__fn=@0x7f4d38400000: (void (nebula::thread::GenericWorker::*)(nebula::thread::GenericWorker * const)) 0x3bbdc3c ) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #31 0x0000000003bc1ad6 in std::_Bind::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f4d38400000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #32 0x0000000003bc1829 in std::_Bind::operator()<, void>() (this=0x7f4d38400000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #33 0x0000000003bc143f in std::_Function_handler >::_M_invoke(std::_Any_data const&) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #34 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d391fcb60) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #35 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #36 0x0000000003bc0398 in std::__invoke_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(std::__invoke_other, void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__f=@0x7f4d5286c2f0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #37 0x0000000003bbf6e3 in std::__invoke, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind >(void (*&&)(std::__cxx11::basic_string, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >&&, std::_Bind&&) (__fn=@0x7f4d5286c2f0: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #38 0x0000000003bc1dc6 in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x7f4d5286c2b8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #39 0x0000000003bc1d5d in std::thread::_Invoker, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > >::operator()() (this=0x7f4d5286c2b8) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #40 0x0000000003bc1d3c in std::thread::_State_impl, std::allocator > const&, std::function const&), std::__cxx11::basic_string, std::allocator >, std::_Bind > > >::_M_run() (this=0x7f4d5286c2b0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #41 0x0000000004b4043f in execute_native_thread_routine () #42 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #43 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 51 (Thread 0x7f4d39fff700 (LWP 3413748) "IOThreadPool8"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d39ffbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cf0086200, req=..., serializedRequest=..., ctx=0x7f4d39245010, eb=0x7f4d39205000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cf0086200, req=..., serializedRequest=..., ctx=0x7f4d39245010, eb=0x7f4d39205000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cf0086200, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d39245010, eb=0x7f4d39205000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cf0086200, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d39245010, eb=0x7f4d39205000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cf0086200, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d39245010, eb=0x7f4d39205000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52832350) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52832350) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52832350) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52832350) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52832340) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 50 (Thread 0x7f4d3adff700 (LWP 3413747) "IOThreadPool7"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3adfbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d42ba7fa0, req=..., serializedRequest=..., ctx=0x7f4d3a0d1210, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d42ba7fa0, req=..., serializedRequest=..., ctx=0x7f4d3a0d1210, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d42ba7fa0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3a0d1210, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d42ba7fa0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3a0d1210, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d42ba7fa0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3a0d1210, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d528322e0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d528322e0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d528322e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d528322e0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d528322d0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 49 (Thread 0x7f4d3bbff700 (LWP 3413746) "IOThreadPool6"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3bbfbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d3ae74ac0, req=..., serializedRequest=..., ctx=0x7f4d3aef2910, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d3ae74ac0, req=..., serializedRequest=..., ctx=0x7f4d3aef2910, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d3ae74ac0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3aef2910, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d3ae74ac0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3aef2910, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d3ae74ac0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3aef2910, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52832200) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52832200) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52832200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52832200) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d528321f0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 48 (Thread 0x7f4d3c9ff700 (LWP 3413745) "IOThreadPool5"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3c9fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cffcfa220, req=..., serializedRequest=..., ctx=0x7f4d3bd1b310, eb=0x7f4d3bc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cffcfa220, req=..., serializedRequest=..., ctx=0x7f4d3bd1b310, eb=0x7f4d3bc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cffcfa220, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3bd1b310, eb=0x7f4d3bc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cffcfa220, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3bd1b310, eb=0x7f4d3bc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cffcfa220, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3bd1b310, eb=0x7f4d3bc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec0d5 in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52832120) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52832120) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52832120) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52832120) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52832110) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 47 (Thread 0x7f4d3d7ff700 (LWP 3413744) "IOThreadPool4"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3d7fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d02d30d00, req=..., serializedRequest=..., ctx=0x7f4d3cab0910, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d02d30d00, req=..., serializedRequest=..., ctx=0x7f4d3cab0910, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d02d30d00, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3cab0910, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d02d30d00, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3cab0910, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d02d30d00, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3cab0910, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52832040) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52832040) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52832040) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52832040) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52832030) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 46 (Thread 0x7f4d3e5ff700 (LWP 3413743) "IOThreadPool3"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3e5fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cef1c9960, req=..., serializedRequest=..., ctx=0x7f4d3d886110, eb=0x7f4d3d804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cef1c9960, req=..., serializedRequest=..., ctx=0x7f4d3d886110, eb=0x7f4d3d804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cef1c9960, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3d886110, eb=0x7f4d3d804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cef1c9960, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3d886110, eb=0x7f4d3d804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cef1c9960, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3d886110, eb=0x7f4d3d804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52831fd0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52831fd0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52831fd0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52831fd0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52831fc0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 45 (Thread 0x7f4d3f5ff700 (LWP 3413742) "IOThreadPool2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d3f5fbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4d4c451200, req=..., serializedRequest=..., ctx=0x7f4d3e6d7210, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4d4c451200, req=..., serializedRequest=..., ctx=0x7f4d3e6d7210, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4d4c451200, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d3e6d7210, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4d4c451200, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d3e6d7210, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4d4c451200, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d3e6d7210, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52831ef0) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52831ef0) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52831ef0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52831ef0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52831ee0) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 44 (Thread 0x7f4d401ff700 (LWP 3413741) "tsc-calibrator"): #0 0x00007f4d52eac8ed in nanosleep () from /lib64/libc.so.6 #1 0x00007f4d52eac784 in sleep () from /lib64/libc.so.6 #2 0x0000000003bb1809 in nebula::time::TscHelper::::operator()(void) const (__closure=0x7f4d3ebff000) at /root/src/wwl/nebula/src/common/time/detail/TscHelper.cpp:27 #3 0x0000000003bb2d6e in std::__invoke_impl&>(std::__invoke_other, nebula::time::TscHelper:: &) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #4 0x0000000003bb2d45 in std::__invoke&>(nebula::time::TscHelper:: &) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #5 0x0000000003bb2d10 in std::_Bind()>::__call(std::tuple<> &&, std::_Index_tuple<>) (this=0x7f4d3ebff000, __args=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:467 #6 0x0000000003bb2c35 in std::_Bind()>::operator()<>(void) (this=0x7f4d3ebff000) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/functional:551 #7 0x0000000003bb2a4b in std::_Function_handler()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:316 #8 0x0000000002bd9f74 in std::function::operator()() const (this=0x7f4d401fcb70) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_function.h:706 #9 0x0000000003b76719 in nebula::thread::NamedThread::hook(std::__cxx11::basic_string, std::allocator > const&, std::function const&) (name=..., f=...) at /root/src/wwl/nebula/src/common/thread/NamedThread.h:59 #10 0x0000000003bb26ba in std::__invoke_impl&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> >(std::__invoke_other, void (*&&)(const std::__cxx11::basic_string, std::allocator > &, const std::function &)) (__f=@0x7f4d402091b8: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #11 0x0000000003bb2247 in std::__invoke&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> >(void (*&&)(const std::__cxx11::basic_string, std::allocator > &, const std::function &)) (__fn=@0x7f4d402091b8: 0x3b766df , std::allocator > const&, std::function const&)>) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #12 0x0000000003bb2ec0 in std::thread::_Invoker, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > >::_M_invoke<0, 1, 2>(std::_Index_tuple<0, 1, 2>) (this=0x7f4d40209188) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #13 0x0000000003bb2e57 in std::thread::_Invoker, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > >::operator()(void) (this=0x7f4d40209188) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #14 0x0000000003bb2e36 in std::thread::_State_impl, std::allocator >&, const std::function&), std::__cxx11::basic_string, std::allocator >, std::_Bind()> > > >::_M_run(void) (this=0x7f4d40209180) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #15 0x0000000004b4043f in execute_native_thread_routine () #16 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 43 (Thread 0x7f4d40fff700 (LWP 3413740) "IOThreadPool1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d40ffbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cf104e7a0, req=..., serializedRequest=..., ctx=0x7f4d402de110, eb=0x7f4d40204000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cf104e7a0, req=..., serializedRequest=..., ctx=0x7f4d402de110, eb=0x7f4d40204000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cf104e7a0, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d402de110, eb=0x7f4d40204000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cf104e7a0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d402de110, eb=0x7f4d40204000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cf104e7a0, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d402de110, eb=0x7f4d40204000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52831e80) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52831e80) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52831e80) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52831e80) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52831e70) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x7f4d41dff700 (LWP 3413739) "IOThreadPool0"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d41dfbbb8, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d557a0 in nebula::raftex::RaftPart::processHeartbeatRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1656 #7 0x0000000003d955dc in nebula::raftex::RaftexService::async_eb_heartbeat (this=0x7f4d528323b0, callback=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:220 #8 0x0000000003e2b971 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_heartbeat (this=0x7f4cf0c8b180, req=..., serializedRequest=..., ctx=0x7f4d410f5810, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:220 #9 0x0000000003e2742a in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::setUpAndProcess_heartbeat (this=0x7f4cf0c8b180, req=..., serializedRequest=..., ctx=0x7f4d410f5810, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:198 #10 0x0000000003e294eb in apache::thrift::detail::ap::process_pmap (proc=0x7f4cf0c8b180, pmap=..., req=..., serializedRequest=..., ctx=0x7f4d410f5810, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:891 #11 0x0000000003e266f3 in apache::thrift::detail::ap::process (processor=0x7f4cf0c8b180, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, ctx=0x7f4d410f5810, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:919 #12 0x0000000003e2539e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::processSerializedRequest (this=0x7f4cf0c8b180, req=..., serializedRequest=..., protType=apache::thrift::protocol::T_COMPACT_PROTOCOL, context=0x7f4d410f5810, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:102 #13 0x00000000042f8644 in apache::thrift::Cpp2Connection::requestReceived(std::unique_ptr >&&) () #14 0x00000000043042cc in apache::thrift::HeaderServerChannel::messageReceived(std::unique_ptr >&&, std::unique_ptr >&&) () #15 0x00000000042c54f6 in apache::thrift::Cpp2Channel::read(wangle::HandlerContext >, apache::thrift::transport::THeader*> >*, std::pair >, std::unique_ptr > >) () #16 0x00000000042d2c73 in wangle::ContextImpl::read(std::pair >, std::unique_ptr > >) () #17 0x00000000042d2b65 in wangle::ContextImpl::fireRead(std::pair >, std::unique_ptr > >) () #18 0x00000000042d37e5 in apache::thrift::FramingHandler::read(wangle::HandlerContext >, std::unique_ptr > >, std::unique_ptr > >*, folly::IOBufQueue&) () #19 0x00000000042cb56d in non-virtual thunk to wangle::ContextImpl::read(folly::IOBufQueue&) () #20 0x00000000042cb450 in wangle::ContextImpl::fireRead(folly::IOBufQueue&) () #21 0x00000000045db8ff in folly::AsyncSocket::handleRead() () #22 0x00000000045d2a00 in folly::AsyncSocket::ioReady(unsigned short) () #23 0x00000000046c284f in ?? () #24 0x00000000046c316f in event_base_loop () #25 0x00000000045ec7fe in folly::EventBase::loopBody(int, bool) () #26 0x00000000045ecca6 in folly::EventBase::loop() () #27 0x00000000045ee756 in folly::EventBase::loopForever() () #28 0x0000000004578389 in folly::IOThreadPoolExecutor::threadRun(std::shared_ptr) () #29 0x0000000004585f59 in void folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) () #30 0x0000000002a3e85c in folly::detail::function::FunctionTraits::operator()() (this=0x7f4d52831e10) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:400 #31 0x0000000002a4ea06 in folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f4d52831e10) at /root/src/wwl/nebula/build/third-party/install/include/folly/executors/thread_factory/NamedThreadFactory.h:40 #32 0x0000000002a60400 in std::__invoke_impl&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__f=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:60 #33 0x0000000002a5994a in std::__invoke&&)::{lambda()#1}>(folly::NamedThreadFactory::newThread(folly::Function&&)::{lambda()#1}&&) (__fn=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/invoke.h:95 #34 0x0000000002aa52c2 in std::thread::_Invoker&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f4d52831e10) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:234 #35 0x0000000002aa47b7 in std::thread::_Invoker&&)::{lambda()#1}> >::operator()() (this=0x7f4d52831e10) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:243 #36 0x0000000002aa1a54 in std::thread::_State_impl&&)::{lambda()#1}> > >::_M_run() (this=0x7f4d52831e00) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/thread:186 #37 0x0000000004b4043f in execute_native_thread_routine () #38 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 41 (Thread 0x7f4d42afe700 (LWP 3413738) "executor-pri4-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb3c0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb3c0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb3c0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528542c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 40 (Thread 0x7f4d42dff700 (LWP 3413737) "executor-pri4-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb3a8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb3a8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb3a8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528542c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x7f4d437ff700 (LWP 3413736) "executor-pri3-3"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadba50, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadba50, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadba50) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 38 (Thread 0x7f4d444fd700 (LWP 3413735) "executor-pri3-3"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d444f9f58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d444fa5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d444fa588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d444fa538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d444fa7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d02d2e980, req=..., serializedRequest=..., ctx=0x7f4d2d6dae10, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cbf46c640, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cf9aa8990) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x7f4d449fd700 (LWP 3413734) "executor-pri3-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadc128, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadc128, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadc128) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x7f4d445fe700 (LWP 3413733) "executor-pri3-3"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadd9b8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadd9b8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadd9b8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x7f4d44afe700 (LWP 3413732) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d44afaf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d44afb5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d44afb588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d44afb538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d44afb7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d02d2e980, req=..., serializedRequest=..., ctx=0x7f4d2d694810, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d2d6c4d80, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdb097110) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 34 (Thread 0x7f4d44bff700 (LWP 3413731) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d44bfbf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d44bfc5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d44bfc588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d44bfc538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d44bfc7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d23285360, req=..., serializedRequest=..., ctx=0x7f4d3cab0c10, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cdbea8900, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdbe3d910) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x7f4d45cfe700 (LWP 3413730) "executor-pri3-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4daddbf8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4daddbf8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4daddbf8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 32 (Thread 0x7f4d45dff700 (LWP 3413729) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d45dfbf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d45dfc5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d45dfc588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d45dfc538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d45dfc7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d42ba7fa0, req=..., serializedRequest=..., ctx=0x7f4d3a0d1510, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cab506e80, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3a143410) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 31 (Thread 0x7f4d463ff700 (LWP 3413728) "executor-pri3-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadc110, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadc110, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadc110) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 30 (Thread 0x7f4d46bfd700 (LWP 3413727) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d46bf9f58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d46bfa5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d46bfa588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d46bfa538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d46bfa7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4cef2038a0, req=..., serializedRequest=..., ctx=0x7f4d3e6ef610, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4ce5cfa380, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4ce59c1b90) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7f4d46cfe700 (LWP 3413726) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d46cfb138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d46cfb5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d46cfb5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d46cfb568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d46cfb7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4cf2532f60, req=..., serializedRequest=..., ctx=0x7f4d2fd35810, eb=0x7f4d2fc04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cbf829e80, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4ccde8d210) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7f4d46fff700 (LWP 3413725) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d46ffbf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d46ffc5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d46ffc588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d46ffc538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d46ffc7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d4c451200, req=..., serializedRequest=..., ctx=0x7f4d3e6d7e10, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cd8df4080, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3e6be110) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7f4d475ff700 (LWP 3413724) "executor-pri3-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadbae0, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadbae0, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadbae0) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7f4d4857d700 (LWP 3413723) "executor-pri3-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadc530, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadc530, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadc530) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7f4d4887e700 (LWP 3413722) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4887b138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d4887b5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4887b5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d4887b568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4887b7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4d0491bae0, req=..., serializedRequest=..., ctx=0x7f4d30ae0b10, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cbf2260c0, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdced6810) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7f4d4867e700 (LWP 3413721) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4867b138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d4867b5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4867b5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d4867b568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4867b7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4cffcfac40, req=..., serializedRequest=..., ctx=0x7f4d3a0a7910, eb=0x7f4d3a004000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cab9a0400, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cd8ffc110) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7f4d48dff700 (LWP 3413720) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d48dfbf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d48dfc5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d48dfc588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d48dfc538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d48dfc7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4cf1903c00, req=..., serializedRequest=..., ctx=0x7f4d318ac110, eb=0x7f4d31804000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cce726440, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3180d110) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7f4d499fd700 (LWP 3413719) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d499f9f58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d499fa5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d499fa588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d499fa538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d499fa7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4cf257c0a0, req=..., serializedRequest=..., ctx=0x7f4d2d7acc10, eb=0x7f4d2d604000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d2d6c4d40, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdb097d10) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7f4d49afe700 (LWP 3413718) "executor-pri3-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadd5f8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadd5f8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadd5f8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7f4d49dff700 (LWP 3413717) "executor-pri3-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadc368, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadc368, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadc368) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7f4d4b7fd700 (LWP 3413716) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4b7fa138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d4b7fa5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4b7fa5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d4b7fa568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4b7fa7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4d23285680, req=..., serializedRequest=..., ctx=0x7f4d3af2b010, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cd97dcf00, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdac05190) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7f4d4b8fe700 (LWP 3413715) "executor-pri3-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadbfa8, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadbfa8, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadbfa8) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7f4d4bbff700 (LWP 3413714) "executor-pri3-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadd970, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadd970, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadd970) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7f4d4c1ff700 (LWP 3413713) "executor-pri3-9"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4c1fbf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d4c1fc5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4c1fc588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d4c1fc538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4c1fc7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d4c451200, req=..., serializedRequest=..., ctx=0x7f4d3e66c510, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d3e6b9340, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3e6bc190) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7f4d4d2fe700 (LWP 3413712) "executor-pri3-8"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4d2faf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d4d2fb5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4d2fb588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d4d2fb538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4d2fb7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d02d30d00, req=..., serializedRequest=..., ctx=0x7f4d3caff610, eb=0x7f4d3ca05000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d3cab7b40, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3caf9190) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7f4d4d3ff700 (LWP 3413711) "executor-pri3-7"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb828, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb828, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb828) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52853cc0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7f4d4d9d9700 (LWP 3413710) "executor-pri3-6"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4d9d5f58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d4d9d65c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4d9d6588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d4d9d6538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4d9d67b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d23238020, req=..., serializedRequest=..., ctx=0x7f4d410e9210, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4ccca21dc0, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d410d0810) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7f4d4dada700 (LWP 3413709) "executor-pri3-5"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4dad6f58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d4dad75c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4dad7588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d4dad7538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4dad77b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4cf4069140, req=..., serializedRequest=..., ctx=0x7f4d30b1d910, eb=0x7f4d30a04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cac405540, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdcc83210) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7f4d4f9ff700 (LWP 3413708) "executor-pri3-1"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d4f9fc138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d4f9fc5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d4f9fc5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d4f9fc568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d4f9fc7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4d3ae74ac0, req=..., serializedRequest=..., ctx=0x7f4d3aed9810, eb=0x7f4d3ae04000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d3ae539c0, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4cdac82f10) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7f4d503fd700 (LWP 3413707) "executor-pri3-3"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d503fa138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d503fa5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d503fa5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d503fa568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d503fa7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4cef2038a0, req=..., serializedRequest=..., ctx=0x7f4d3e66b010, eb=0x7f4d3e605000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4d3e6b9380, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d3e6bc010) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7f4d504fe700 (LWP 3413706) "executor-pri3-2"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d504faf58, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d530bd in nebula::raftex::RaftPart::processAppendLogRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1373 #7 0x0000000003d9540c in nebula::raftex::RaftexService::appendLog (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:194 #8 0x0000000003e24bc0 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AppendLogResponse &) const (__closure=0x7f4d504fb5c0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #9 0x0000000003e27e1d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e25c85 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d504fb588) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2c352 in folly::::operator()(void) (__closure=0x7f4d504fb538) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e34801 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e30585 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2c388 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e27e7c in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25cba in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e24c02 in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:44 #18 0x0000000003e24cdf in nebula::raftex::cpp2::RaftexServiceSvIf::future_appendLog (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:51 #19 0x0000000003e24da2 in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d504fb7b0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:57 #20 0x0000000003e280dd in folly::makeFutureWith >, const nebula::raftex::cpp2::AppendLogRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e25e70 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e26062 in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AppendLogRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24e1e in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_appendLog (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:55 #24 0x0000000003e2b23e in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_appendLog (this=0x7f4d23238020, req=..., serializedRequest=..., ctx=0x7f4d410c3310, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:111 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cbc772600, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d41101990) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7f4d507ff700 (LWP 3413705) "executor-pri3-4"): #0 0x00007f4d531c354d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f4d531bee9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f4d531bed68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000002a3863a in __gthread_mutex_lock (__mutex=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/x86_64-vesoft-linux/bits/gthr-default.h:748 #4 0x0000000002a3893c in std::mutex::lock (this=0x7f4d384be150) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:103 #5 0x0000000002a3e41b in std::lock_guard::lock_guard (this=0x7f4d507fc138, __m=...) at /data/vesoft/toolset/gcc/7.5.0/include/c++/7.5.0/bits/std_mutex.h:162 #6 0x0000000003d51c1a in nebula::raftex::RaftPart::processAskForVoteRequest (this=0x7f4d384be010, req=..., resp=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftPart.cpp:1248 #7 0x0000000003d9533c in nebula::raftex::RaftexService::askForVote (this=0x7f4d528323b0, resp=..., req=...) at /root/src/wwl/nebula/src/kvstore/raftex/RaftexService.cpp:183 #8 0x0000000003e248da in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(nebula::raftex::cpp2::AskForVoteResponse &) const (__closure=0x7f4d507fc5f0, _return=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #9 0x0000000003e2759d in apache::thrift::detail::si::returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1204 #10 0x0000000003e257e3 in apache::thrift::detail::si::::operator()(void) const (__closure=0x7f4d507fc5b8) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #11 0x0000000003e2bbb3 in folly::::operator()(void) (__closure=0x7f4d507fc568) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #12 0x0000000003e342a3 in folly::makeTryWithNoUnwrap]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:244 #13 0x0000000003e2ff71 in folly::makeTryWith]::]:: >(folly:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Try-inl.h:270 #14 0x0000000003e2bbe4 in folly::makeSemiFutureWith]:: >(apache::thrift::detail::si:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:608 #15 0x0000000003e275de in apache::thrift::detail::si::semifuture]:: >(apache::thrift::detail::si:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1198 #16 0x0000000003e25813 in apache::thrift::detail::si::semifuture_returning >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1210 #17 0x0000000003e2491c in nebula::raftex::cpp2::RaftexServiceSvIf::semifuture_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:23 #18 0x0000000003e249f9 in nebula::raftex::cpp2::RaftexServiceSvIf::future_askForVote (this=0x7f4d528323b0, p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:30 #19 0x0000000003e24abc in nebula::raftex::cpp2::RaftexServiceSvIf::::operator()(void) const (__closure=0x7f4d507fc7e0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:36 #20 0x0000000003e279cb in folly::makeFutureWith >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (func=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/futures/Future-inl.h:1241 #21 0x0000000003e259c8 in apache::thrift::detail::si::makeFutureWithAndMaybeReschedule >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::detail::si::CallbackBase &, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1240 #22 0x0000000003e25bba in apache::thrift::detail::si::async_tm >, const nebula::raftex::cpp2::AskForVoteRequest&):: >(apache::thrift::ServerInterface *, apache::thrift::detail::si::CallbackPtr, nebula::raftex::cpp2::RaftexServiceSvIf:: &&) (si=0x7f4d528323b8, callback=..., f=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/GeneratedCodeHelper.h:1263 #23 0x0000000003e24b38 in nebula::raftex::cpp2::RaftexServiceSvIf::async_tm_askForVote (this=0x7f4d528323b0, callback=..., p_req=...) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.cpp:34 #24 0x0000000003e2ae92 in nebula::raftex::cpp2::RaftexServiceAsyncProcessor::process_askForVote (this=0x7f4cf0c8b180, req=..., serializedRequest=..., ctx=0x7f4d410c2a10, eb=0x7f4d41005000, tm=0x7f4d528141d0) at /root/src/wwl/nebula/build/src/interface/gen-cpp2/RaftexService.tcc:54 #25 0x0000000003e2dba1 in apache::thrift::GeneratedAsyncProcessor::makeEventTaskForRequest(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}::operator()(std::unique_ptr) (__closure=0x7f4cdb6693c0, rq=...) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:698 #26 0x0000000003e3f353 in folly::detail::function::FunctionTraits)>::callBig(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*, apache::thrift::Tile*)::{lambda(std::unique_ptr)#1}>(std::unique_ptr&&, folly::detail::function::Data&) (args#0=..., p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:385 #27 0x00000000042032ea in apache::thrift::EventTask::run() () #28 0x0000000003e29913 in apache::thrift::GeneratedAsyncProcessor::processInThread(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}::operator()() const (__closure=0x7f4d410cf910) at /root/src/wwl/nebula/build/third-party/install/include/thrift/lib/cpp2/async/AsyncProcessor.h:773 #29 0x0000000003e35803 in folly::detail::function::FunctionTraits::callSmall(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*, apache::thrift::RpcKind, void (nebula::raftex::cpp2::RaftexServiceAsyncProcessor::*)(std::unique_ptr, apache::thrift::SerializedRequest&&, apache::thrift::Cpp2RequestContext*, folly::EventBase*, apache::thrift::concurrency::ThreadManager*), nebula::raftex::cpp2::RaftexServiceAsyncProcessor*)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/Function.h:371 #30 0x000000000420a359 in apache::thrift::concurrency::FunctionRunner::run() () #31 0x0000000004339b82 in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #32 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #33 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7f4d50ffd700 (LWP 3413704) "executor-pri2-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb060, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb060, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb060) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528536c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7f4d510fe700 (LWP 3413703) "executor-pri2-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb078, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb078, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb078) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528536c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7f4d511ff700 (LWP 3413702) "executor-pri1-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb090, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb090, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb090) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528530c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7f4d515ff700 (LWP 3413701) "executor-pri1-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb018, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb018, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb018) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d528530c0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7f4d52de6700 (LWP 3413700) "executor-pri0-2"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb030, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb030, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb030) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52852ac0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7f4d53d00700 (LWP 3413699) "executor-pri0-1"): #0 0x00007f4d52edfd19 in syscall () from /lib64/libc.so.6 #1 0x000000000410c141 in folly::detail::futexWaitImpl(std::atomic const*, unsigned int, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int) () #2 0x0000000002a6cf52 in folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=0x7f4d4dadb048, expected=2, deadline=..., waitMask=4294967295) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/Futex-inl.h:119 #3 0x0000000002a6764d in folly::detail::MemoryIdler::futexWaitUntil, std::chrono::time_point > >, std::chrono::duration > > (fut=..., expected=2, deadline=..., waitMask=4294967295, idleTimeout=..., stackToRetain=1024, timeoutVariationFrac=0.5) at /root/src/wwl/nebula/build/third-party/install/include/folly/detail/MemoryIdler.h:164 #4 0x0000000002aa86c9 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f4d4dadb048, deadline=..., opt=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:312 #5 0x0000000002aa5d28 in folly::SaturatingSemaphore::try_wait_until > > (opt=..., deadline=..., this=0x7f4d4dadb048) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/SaturatingSemaphore.h:173 #6 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f4d52852ac0, deadline=...) at /root/src/wwl/nebula/build/third-party/install/include/folly/synchronization/LifoSem.h:512 #7 0x0000000004330f81 in apache::thrift::concurrency::ThreadManager::Impl::waitOnTask() () #8 0x0000000004339a8c in apache::thrift::concurrency::ThreadManager::Impl::Worker::run() () #9 0x000000000433db48 in apache::thrift::concurrency::PthreadThread::threadMain(void*) () #10 0x00007f4d531bcea5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f4d52ee59fd in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f4d53d049c0 (LWP 3413698) "nebula-storaged"): #0 0x00007f4d531be017 in pthread_join () from /lib64/libpthread.so.0 #1 0x0000000004b404d3 in std::thread::join() () #2 0x0000000002a59090 in nebula::storage::StorageServer::waitUntilStop (this=0x7f4d5284f180) at /root/src/wwl/nebula/src/storage/StorageServer.cpp:310 #3 0x0000000002a3d9f4 in main (argc=1, argv=0x7ffe727c2568) at /root/src/wwl/nebula/src/daemons/StorageDaemon.cpp:158