PWL cache unwindpmp profiling

nopwl-1.log

sample: 499

Thread 3021 (CPU 0/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3022 (CPU 1/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3023 (CPU 2/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3024 (CPU 3/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3020 (IO mon_iothread) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% iothread_run
    + 100.00% g_main_loop_run
     + 100.00% g_hmac_get_digest
      + 100.00% poll

Thread 3009 (ceph_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% ceph::timer<ceph::coarse_mono_clock>::timer_thread()
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3008 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 99.80% pthread_cond_wait
      + 99.80% __futex_abstimed_wait_common

Thread 3007 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 97.00% pthread_cond_wait
     |+ 97.00% __futex_abstimed_wait_common
     + 2.20% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 2.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 2.00% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |  + 1.20% librbd::io::ImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     |  |+ 1.20% librbd::io::ImageRequest<librbd::ImageCtx>::aio_write(librbd::ImageCtx*, librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&)
     |  | + 1.20% librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>::send_request()
     |  |  + 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ObjectDispatcherInterface>::send(librbd::io::ObjectDispatchSpec*)
     |  |  |+ 0.80% librbd::io::ObjectDispatch<librbd::ImageCtx>::write(unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, int, std::optional<unsigned long>, ZTracer::Trace const&, int*, unsigned long*, librbd::io::DispatchResult*, Context**, Context*)
     |  |  ||+ 0.60% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()
     |  |  |||+ 0.40% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |  |  ||||+ 0.20% Throttle::get_or_fail(long)
     |  |  |||||+ 0.20% ceph::common::PerfCounters::set(int, unsigned long)
     |  |  ||||+ 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |  |  |||| + 0.20% AsyncConnection::send_message(Message*)
     |  |  ||||  + 0.20% ProtocolV2::send_message(Message*)
     |  |  ||||   + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     |  |  ||||    + 0.20% EventCenter::wakeup()
     |  |  ||||     + 0.20% __write
     |  |  |||+ 0.20% operator delete(void*)
     |  |  ||+ 0.20% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::send()
     |  |  || + 0.20% __pthread_rwlock_wrlock
     |  |  |+ 0.20% librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::write(unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, int, std::optional<unsigned long>, ZTracer::Trace const&, int*, unsigned long*, librbd::io::DispatchResult*, Context**, Context*)
     |  |  + 0.20% librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>::prune_object_extents(boost::container::small_vector<striper::LightweightObjectExtent, 4ul, void, void>*) const
     |  + 0.60% librbd::io::ImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     |  |+ 0.40% librbd::io::ImageReadRequest<librbd::ImageCtx>::send_request()
     |  ||+ 0.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ObjectDispatcherInterface>::send(librbd::io::ObjectDispatchSpec*)
     |  || + 0.40% librbd::io::ObjectDispatch<librbd::ImageCtx>::read(unsigned long, std::vector<librbd::io::ReadExtent, std::allocator<librbd::io::ReadExtent> >*, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long*, int*, librbd::io::DispatchResult*, Context**, Context*)
     |  ||  + 0.40% librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()
     |  ||   + 0.20% librbd::ObjectMap<librbd::ImageCtx>::object_may_exist(unsigned long) const
     |  ||   + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |  ||    + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |  ||     + 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     |  |+ 0.20% librbd::io::ImageRequest<librbd::ImageCtx>::send()
     |  | + 0.20% librbd::io::ImageRequest<librbd::ImageCtx>::update_timestamp()
     |  |  + 0.20% clock_gettime
     |  |   + 0.20% ???
     |  + 0.20% std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_add_ref_copy()
     + 0.20% __GI___pthread_mutex_unlock_usercnt
     |+ 0.20% __GI___lll_lock_wake
     + 0.20% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.20% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
     | + 0.20% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
     + 0.20% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}>(librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}>(librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
      + 0.20% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
       + 0.20% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
        + 0.20% Context::complete(int)
         + 0.20% librbd::io::AioCompletion::complete_request(long)
          + 0.20% librbd::io::AioCompletion::complete()
           + 0.20% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
            + 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
             + 0.20% aio_bh_schedule_oneshot_full
              + 0.20% event_notifier_set
               + 0.20% __write

Thread 2999 (log) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::logging::Log::entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

Thread 3010 (ms_dispatch) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::DispatchThread::entry()
   + 100.00% DispatchQueue::entry()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3011 (ms_local) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::LocalDeliveryThread::entry()
   + 100.00% DispatchQueue::run_local_delivery()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3000 (msgr-worker-0) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 96.20% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 96.20% epoll_wait
     + 2.20% ProtocolV2::write_event()
     |+ 1.20% ProtocolV2::write_message(Message*, bool)
     ||+ 0.80% AsyncConnection::_try_send(bool)
     |||+ 0.80% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
     ||| + 0.60% sendmsg
     ||| + 0.20% ceph::buffer::v15_2_0::list::buffers_t::clear_and_dispose()
     |||  + 0.20% ceph::buffer::v15_2_0::ptr::release()
     ||+ 0.20% ceph::msgr::v2::FrameAssembler::assemble_frame(ceph::msgr::v2::Tag, ceph::buffer::v15_2_0::list*, unsigned short const*, unsigned long)
     || + 0.20% ceph::msgr::v2::FrameAssembler::asm_crc_rev1(ceph::msgr::v2::preamble_block_t const&, ceph::buffer::v15_2_0::list*) const
     ||  + 0.20% ceph::buffer::v15_2_0::list::reserve(unsigned long)
     ||   + 0.20% ceph::buffer::v15_2_0::ptr_node::create_hypercombined(ceph::unique_leakable_ptr<ceph::buffer::v15_2_0::raw>)
     ||    + 0.20% operator new(unsigned long)
     ||     + 0.20% malloc
     |+ 0.60% Message::encode(unsigned long, int, bool)
     ||+ 0.60% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::encode_payload(unsigned long)
     || + 0.20% object_locator_t::encode(ceph::buffer::v15_2_0::list&) const
     || |+ 0.20% ceph::buffer::v15_2_0::ptr::append(char const*, unsigned int)
     || + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     || |+ 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     || | + 0.20% posix_memalign
     || |  + 0.20% malloc
     || |   + 0.20% _int_malloc
     || |    + 0.20% malloc_consolidate
     || + 0.20% ceph::buffer::v15_2_0::ptr::append(char const*, unsigned int)
     ||  + 0.20% ceph::buffer::v15_2_0::ptr::unused_tail_length() const
     |+ 0.20% pthread_mutex_lock
     |+ 0.20% ceph::mono_clock::now()
     | + 0.20% clock_gettime
     + 1.40% AsyncConnection::process()
     |+ 1.40% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
     | + 1.20% ProtocolV2::handle_message()
     | |+ 0.80% DispatchQueue::fast_dispatch(Message*)
     | ||+ 0.80% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
     | || + 0.80% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
     | ||  + 0.80% Objecter::ms_dispatch(Message*)
     | ||   + 0.80% Objecter::handle_osd_op_reply(MOSDOpReply*)
     | ||    + 0.40% ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code>::destroy_defer(std::tuple<boost::system::error_code>&&) [clone .lto_priv.0]
     | ||    |+ 0.40% pthread_cond_signal
     | ||    + 0.20% ceph::common::RefCountedObject::_get() const
     | |+ 0.40% decode_message(ceph::common::CephContext*, int, ceph_msg_header&, ceph_msg_footer&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, boost::intrusive_ptr<Connection>)
     | | + 0.20% Message::set_data(ceph::buffer::v15_2_0::list const&)
     | + 0.20% ProtocolV2::read_frame_segment()
     |  + 0.20% ceph::buffer::v15_2_0::ptr_node::create_hypercombined(ceph::unique_leakable_ptr<ceph::buffer::v15_2_0::raw>)
     |   + 0.20% ceph::buffer::v15_2_0::ptr::ptr(ceph::unique_leakable_ptr<ceph::buffer::v15_2_0::raw>)
     + 0.20% clock_gettime
      + 0.20% ???

Thread 3001 (msgr-worker-1) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 93.80% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 93.80% epoll_wait
     + 2.80% ProtocolV2::write_event()
     |+ 2.20% ProtocolV2::write_message(Message*, bool)
     ||+ 1.80% AsyncConnection::_try_send(bool)
     |||+ 1.80% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
     ||| + 1.80% sendmsg
     ||+ 0.20% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::~MOSDOp()
     |||+ 0.20% ceph::buffer::v15_2_0::raw_combined::~raw_combined()
     ||| + 0.20% mempool::get_pool(mempool::pool_index_t)
     ||+ 0.20% ceph::msgr::v2::FrameAssembler::assemble_frame(ceph::msgr::v2::Tag, ceph::buffer::v15_2_0::list*, unsigned short const*, unsigned long)
     || + 0.20% __memset_evex_unaligned_erms
     |+ 0.60% Message::encode(unsigned long, int, bool)
     | + 0.60% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::encode_payload(unsigned long)
     |  + 0.40% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     |   + 0.40% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     |    + 0.20% posix_memalign
     |    |+ 0.20% malloc
     |    | + 0.20% _int_malloc
     |    |  + 0.20% malloc_consolidate
     |    + 0.20% mempool::get_pool(mempool::pool_index_t)
     + 2.80% AsyncConnection::process()
     |+ 2.00% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
     ||+ 1.00% ProtocolV2::handle_message()
     |||+ 0.40% DispatchQueue::fast_dispatch(Message*)
     ||||+ 0.40% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
     |||| + 0.40% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
     ||||  + 0.40% Objecter::ms_dispatch(Message*)
     ||||   + 0.40% Objecter::handle_osd_op_reply(MOSDOpReply*)
     ||||    + 0.20% ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code>::destroy_defer(std::tuple<boost::system::error_code>&&) [clone .lto_priv.0]
     ||||    |+ 0.20% pthread_cond_signal
     ||||    + 0.20% _init
     |||+ 0.20% ceph::mono_clock::now()
     ||||+ 0.20% _init
     |||+ 0.20% decode_message(ceph::common::CephContext*, int, ceph_msg_header&, ceph_msg_footer&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, boost::intrusive_ptr<Connection>)
     ||| + 0.20% MOSDOpReply::decode_payload()
     |||  + 0.20% Message::decode_trace(ceph::buffer::v15_2_0::list::iterator_impl<true>&, bool)
     |||   + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, char*) [clone .localalias] [clone .lto_priv.0]
     ||+ 0.60% ProtocolV2::read_frame_segment()
     |||+ 0.40% ProtocolV2::read(CtRxNode<ProtocolV2>&, std::unique_ptr<ceph::buffer::v15_2_0::ptr_node, ceph::buffer::v15_2_0::ptr_node::disposer>&&)
     ||||+ 0.20% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
     |||| + 0.20% AsyncConnection::read_until(unsigned int, char*)
     ||||  + 0.20% AsyncConnection::read_bulk(char*, unsigned int)
     ||||   + 0.20% PosixConnectedSocketImpl::read(char*, unsigned long)
     ||||    + 0.20% read
     |||+ 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
     ||| + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
     |||  + 0.20% posix_memalign
     |||   + 0.20% _mid_memalign.isra.0
     ||+ 0.20% ProtocolV2::read_frame()
     |||+ 0.20% ProtocolV2::read(CtRxNode<ProtocolV2>&, std::unique_ptr<ceph::buffer::v15_2_0::ptr_node, ceph::buffer::v15_2_0::ptr_node::disposer>&&)
     ||| + 0.20% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
     |||  + 0.20% AsyncConnection::read_until(unsigned int, char*)
     |||   + 0.20% AsyncConnection::read_bulk(char*, unsigned int)
     |||    + 0.20% PosixConnectedSocketImpl::read(char*, unsigned long)
     |||     + 0.20% read
     ||+ 0.20% ProtocolV2::_handle_read_frame_epilogue_main()
     || + 0.20% ceph::msgr::v2::FrameAssembler::disassemble_segments(ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list*, ceph::buffer::v15_2_0::list&) const
     ||  + 0.20% ceph::msgr::v2::FrameAssembler::disasm_remaining_crc_rev1(ceph::buffer::v15_2_0::list*, ceph::buffer::v15_2_0::list&) const
     ||   + 0.20% ceph::msgr::v2::check_segment_crc(ceph::buffer::v15_2_0::list const&, unsigned int)
     ||    + 0.20% ceph::buffer::v15_2_0::list::crc32c(unsigned int) const
     ||     + 0.20% crc32_iscsi_00
     |+ 0.80% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
     | + 0.80% AsyncConnection::read_until(unsigned int, char*)
     |  + 0.80% AsyncConnection::read_bulk(char*, unsigned int)
     |   + 0.80% PosixConnectedSocketImpl::read(char*, unsigned long)
     |    + 0.80% read
     + 0.20% C_handle_notify::do_request(unsigned long)
     |+ 0.20% read
     + 0.20% __libc_free
      + 0.20% _int_free

Thread 3002 (msgr-worker-2) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 100.00% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
      + 100.00% epoll_wait

Thread 2998 (qemu-system-x86) - 500 samples
+ 100.00% __clone3
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% call_rcu_thread
    + 100.00% qemu_event_wait
     + 100.00% syscall

Thread 2974 (qemu-system-x86) - 500 samples
+ 95.60% _start
|+ 95.60% __libc_start_main
| + 95.60% __libc_start_call_main
|  + 95.60% qemu_default_main
|   + 95.60% qemu_main_loop
|    + 95.60% main_loop_wait
|     + 91.20% qemu_poll_ns
|     |+ 91.20% ppoll
|     + 1.80% g_main_context_dispatch
|     |+ 1.80% aio_ctx_dispatch
|     | + 1.80% aio_dispatch
|     |  + 1.40% aio_dispatch_handler
|     |  |+ 1.00% virtio_queue_notify_vq.part.0
|     |  ||+ 1.00% virtio_blk_handle_vq
|     |  || + 0.80% virtqueue_split_pop
|     |  || |+ 0.40% address_space_cache_init
|     |  || ||+ 0.20% address_space_translate_internal
|     |  || |+ 0.20% virtqueue_map_desc
|     |  || | + 0.20% address_space_map
|     |  || |  + 0.20% memory_region_ref
|     |  || + 0.20% virtio_blk_submit_multireq
|     |  ||  + 0.20% blk_aio_pwritev
|     |  ||   + 0.20% blk_aio_prwv
|     |  |+ 0.20% virtio_queue_host_notifier_read
|     |  | + 0.20% event_notifier_test_and_clear
|     |  |  + 0.20% read
|     |  |   + 0.20% __GI___pthread_enable_asynccancel
|     |  + 0.20% aio_bh_poll
|     |  + 0.20% qemu_lockcnt_inc
|     + 1.20% g_main_context_prepare
|     |+ 0.40% g_mutex_unlock
|     |+ 0.40% g_cond_wait_until
|     | + 0.40% g_hmac_get_digest
|     |  + 0.40% g_hmac_get_digest
|     |   + 0.20% __pthread_getspecific
|     + 1.00% g_main_context_check
|     |+ 0.40% g_mutex_lock
|     |+ 0.20% g_get_monotonic_time
|     ||+ 0.20% g_hmac_get_digest
|     |+ 0.20% qio_channel_fd_source_check
|     |+ 0.20% aio_ctx_check
|     | + 0.20% aio_pending
|     + 0.20% timerlistgroup_deadline_ns
|     |+ 0.20% timerlist_deadline_ns
|     | + 0.20% qemu_clock_get_ns
|     + 0.20% g_main_context_query
+ 4.40% __push___start_context
 + 4.40% coroutine_trampoline
  + 1.60% blk_aio_write_entry
  |+ 1.20% blk_co_do_pwritev_part
  ||+ 1.20% bdrv_co_pwritev_part
  || + 1.20% bdrv_aligned_pwritev
  ||  + 1.20% bdrv_driver_pwritev
  ||   + 1.20% raw_co_pwritev
  ||    + 1.20% bdrv_co_pwritev_part
  ||     + 1.20% bdrv_aligned_pwritev
  ||      + 1.20% bdrv_driver_pwritev
  ||       + 1.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  ||        + 1.20% rbd_aio_writev
  ||         + 0.60% (anonymous namespace)::iovec_to_bufferlist(librbd::ImageCtx*, iovec const*, int, librbd::io::AioCompletion*) [clone .lto_priv.0]
  ||         |+ 0.60% (anonymous namespace)::create_write_raw(librbd::ImageCtx*, char const*, unsigned long, librbd::io::AioCompletion*) [clone .lto_priv.0]
  ||         | + 0.60% ceph::buffer::v15_2_0::copy(char const*, unsigned int)
  ||         |  + 0.40% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
  ||         |  |+ 0.40% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
  ||         |  | + 0.20% mempool::pool_t::adjust_count(long, long)
  ||         |  | + 0.20% posix_memalign
  ||         |  |  + 0.20% malloc
  ||         |  |   + 0.20% _int_malloc
  ||         |  |    + 0.20% unlink_chunk.isra.0
  ||         |  + 0.20% __memcpy_evex_unaligned_erms
  ||         + 0.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  ||         |+ 0.40% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  ||         | + 0.40% librbd::io::QueueImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  ||         |  + 0.20% void boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>::execute<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}> >(boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>&&) const [clone .lto_priv.0]
  ||         |  |+ 0.20% boost::asio::detail::scheduler::post_immediate_completion(boost::asio::detail::scheduler_operation*, bool)
  ||         |  + 0.20% librbd::io::FlushTracker<librbd::ImageCtx>::start_io(unsigned long)
  ||         |   + 0.20% operator new(unsigned long)
  ||         |    + 0.20% malloc
  ||         |     + 0.20% _int_malloc
  ||         |      + 0.20% unlink_chunk.isra.0
  ||         + 0.20% librbd::api::Io<librbd::ImageCtx>::aio_write(librbd::ImageCtx&, librbd::io::AioCompletion*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, int, bool)
  ||          + 0.20% librbd::ImageCtx::get_data_io_context() const
  |+ 0.40% virtio_blk_rw_complete
  | + 0.20% virtio_notify_irqfd
  | |+ 0.20% virtio_should_notify
  | + 0.20% event_notifier_set
  |  + 0.20% __write
  + 1.60% blk_aio_flush_entry
  |+ 1.40% bdrv_co_flush
  ||+ 1.40% bdrv_co_flush
  || + 1.40% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  ||  + 1.00% rbd_aio_flush
  ||  |+ 1.00% librbd::api::Io<librbd::ImageCtx>::aio_flush(librbd::ImageCtx&, librbd::io::AioCompletion*, bool)
  ||  | + 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  ||  |  + 1.00% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  ||  |   + 1.00% librbd::io::QueueImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  ||  |    + 1.00% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
  ||  |     + 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  ||  |      + 0.80% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  ||  |      |+ 0.80% librbd::io::QosImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  ||  |      | + 0.80% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
  ||  |      |  + 0.80% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  ||  |      |   + 0.60% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  ||  |      |   |+ 0.40% librbd::io::ImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  ||  |      |   ||+ 0.40% librbd::io::ImageRequest<librbd::ImageCtx>::aio_flush(librbd::ImageCtx*, librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&)
  ||  |      |   || + 0.20% librbd::io::ImageFlushRequest<librbd::ImageCtx>::send_request()
  ||  |      |   ||  + 0.20% librbd::io::AsyncOperation::flush(Context*)
  ||  |      |   ||   + 0.20% void boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>::execute<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}> >(boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>&&) const [clone .lto_priv.0]
  ||  |      |   ||    + 0.20% pthread_cond_signal
  ||  |      |   |+ 0.20% librbd::io::WriteBlockImageDispatch<librbd::ImageCtx>::process_io(unsigned long, librbd::io::DispatchResult*, Context**, Context*)
  ||  |      |   | + 0.20% operator new(unsigned long)
  ||  |      |   |  + 0.20% malloc
  ||  |      |   + 0.20% __pthread_rwlock_rdlock
  ||  |      + 0.20% __pthread_rwlock_unlock
  ||  + 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  ||  + 0.20% rbd_aio_create_completion
  ||   + 0.20% operator new(unsigned long)
  ||    + 0.20% malloc
  ||     + 0.20% _int_malloc
  |+ 0.20% virtio_blk_flush_complete
  | + 0.20% virtio_blk_req_complete
  |  + 0.20% virtqueue_push
  |   + 0.20% virtqueue_fill
  + 1.20% blk_aio_read_entry
   + 0.80% blk_co_do_preadv_part
   |+ 0.80% bdrv_co_preadv_part
   | + 0.80% bdrv_aligned_preadv
   |  + 0.80% bdrv_driver_preadv
   |   + 0.80% bdrv_co_preadv_part
   |    + 0.80% bdrv_aligned_preadv
   |     + 0.80% bdrv_driver_preadv
   |      + 0.80% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
   |       + 0.60% rbd_aio_readv
   |       |+ 0.60% librbd::api::Io<librbd::ImageCtx>::aio_read(librbd::ImageCtx&, librbd::io::AioCompletion*, unsigned long, unsigned long, librbd::io::ReadResult&&, int, bool)
   |       | + 0.60% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
   |       |  + 0.40% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
   |       |  |+ 0.20% librbd::io::QueueImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
   |       |  ||+ 0.20% void boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>::execute<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}> >(boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>&&) const [clone .lto_priv.0]
   |       |  || + 0.20% pthread_cond_signal
   |       |  |+ 0.20% librbd::ImageCtx::get_effective_image_size(unsigned long) const
   |       |  | + 0.20% __libc_free
   |       |  + 0.20% AsyncOpTracker::start_op()
   |       |   + 0.20% pthread_mutex_lock
   |       + 0.20% rbd_aio_create_completion
   |        + 0.20% _init
   + 0.40% virtio_blk_rw_complete
    + 0.20% virtio_blk_req_complete
    + 0.20% event_notifier_set
     + 0.20% __write

Thread 3014 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3013 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3012 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3015 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3006 (service) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::common::CephContextServiceThread::entry()
   + 100.00% pthread_cond_clockwait
    + 100.00% __futex_abstimed_wait_common

Thread 3016 (taskfin_librbd) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% Finisher::finisher_thread_entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

nopwl-2.log

sample: 499

Thread 3021 (CPU 0/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3022 (CPU 1/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3023 (CPU 2/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3024 (CPU 3/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 3020 (IO mon_iothread) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% iothread_run
    + 100.00% g_main_loop_run
     + 100.00% g_hmac_get_digest
      + 100.00% poll

Thread 3009 (ceph_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% ceph::timer<ceph::coarse_mono_clock>::timer_thread()
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3008 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 99.80% pthread_cond_wait
     |+ 99.80% __futex_abstimed_wait_common
     + 0.20% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
      + 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
       + 0.20% Context::complete(int)
        + 0.20% librbd::io::AioCompletion::complete_request(long)
         + 0.20% librbd::io::AioCompletion::complete()
          + 0.20% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
           + 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
            + 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
             + 0.20% rbd_aio_get_return_value

Thread 3007 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 96.80% pthread_cond_wait
     |+ 96.60% __futex_abstimed_wait_common
     |+ 0.20% __GI___pthread_mutex_unlock_usercnt
     | + 0.20% __GI___lll_lock_wake
     + 1.40% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 1.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 1.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     | |+ 1.00% librbd::io::ImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     | ||+ 1.00% librbd::io::ImageRequest<librbd::ImageCtx>::aio_write(librbd::ImageCtx*, librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&)
     | || + 1.00% librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>::send_request()
     | ||  + 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ObjectDispatcherInterface>::send(librbd::io::ObjectDispatchSpec*)
     | ||   + 1.00% librbd::io::ObjectDispatch<librbd::ImageCtx>::write(unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, int, std::optional<unsigned long>, ZTracer::Trace const&, int*, unsigned long*, librbd::io::DispatchResult*, Context**, Context*)
     | ||    + 0.80% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()
     | ||    |+ 0.60% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     | ||    ||+ 0.60% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     | ||    || + 0.40% AsyncConnection::send_message(Message*)
     | ||    || |+ 0.40% ProtocolV2::send_message(Message*)
     | ||    || | + 0.40% EventCenter::dispatch_event_external(EventCallback*)
     | ||    || |  + 0.20% EventCenter::wakeup()
     | ||    || |  |+ 0.20% __write
     | ||    || |  + 0.20% pthread_mutex_lock
     | ||    || + 0.20% Objecter::_get_session(int, Objecter::OSDSession**, ceph::shunique_lock<std::shared_mutex>&) [clone .isra.0]
     | ||    ||  + 0.20% ceph::common::RefCountedObject::_get() const
     | ||    |+ 0.20% neorados::RADOS::execute(neorados::Object const&, neorados::IOContext const&, neorados::WriteOp&&, std::unique_ptr<ceph::async::Completion<void (boost::system::error_code), void>, std::default_delete<ceph::async::Completion<void (boost::system::error_code), void> > >, unsigned long*, blkin_trace_info const*) [clone .constprop.0]
     | ||    + 0.20% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::send()
     | ||     + 0.20% std::__shared_mutex_pthread::lock_shared()
     | ||      + 0.20% __pthread_rwlock_rdlock
     | |+ 0.20% librbd::io::ImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     | | + 0.20% librbd::io::ImageReadRequest<librbd::ImageCtx>::send_request()
     | |  + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ObjectDispatcherInterface>::send(librbd::io::ObjectDispatchSpec*)
     | |   + 0.20% librbd::io::ObjectDispatch<librbd::ImageCtx>::read(unsigned long, std::vector<librbd::io::ReadExtent, std::allocator<librbd::io::ReadExtent> >*, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long*, int*, librbd::io::DispatchResult*, Context**, Context*)
     | |    + 0.20% librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()
     | |     + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     | |      + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     | |       + 0.20% AsyncConnection::send_message(Message*)
     | |        + 0.20% ProtocolV2::send_message(Message*)
     | |         + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     | |          + 0.20% EventCenter::wakeup()
     | |           + 0.20% __write
     | + 0.20% _init
     + 0.60% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}>(librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}>(librbd::io::ObjectReadRequest<librbd::ImageCtx>::read_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.40% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
     | + 0.40% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
     |  + 0.40% Context::complete(int)
     |   + 0.40% librbd::io::AioCompletion::complete_request(long)
     |    + 0.20% std::_Rb_tree<unsigned long, std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> >, std::_Select1st<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > > >::_M_erase(std::_Rb_tree_node<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > >*) [clone .isra.0]
     |    |+ 0.20% _init
     |    + 0.20% librbd::io::AioCompletion::complete()
     |     + 0.20% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     |      + 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |       + 0.20% aio_bh_schedule_oneshot_full
     + 0.60% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.40% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
     ||+ 0.40% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
     || + 0.40% Context::complete(int)
     ||  + 0.40% LambdaContext<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::register_in_flight_request(unsigned long, utime_t const&, Context**)::{lambda(int)#1}>::finish(int)
     ||   + 0.40% Context::complete(int)
     ||    + 0.40% librbd::io::AioCompletion::complete_request(long)
     ||     + 0.40% librbd::io::AioCompletion::complete()
     ||      + 0.40% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     ||       + 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||        + 0.20% librbd::io::AioCompletion::notify_callbacks_complete()
     ||        |+ 0.20% Context::complete(int)
     ||        | + 0.20% Context::complete(int)
     ||        |  + 0.20% LambdaContext<librbd::io::QosImageDispatch<librbd::ImageCtx>::needs_throttle(bool, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > > const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)::{lambda(int)#1}>::finish(int)
     ||        |   + 0.20% librbd::io::FlushTracker<librbd::ImageCtx>::finish_io(unsigned long)
     ||        |    + 0.20% std::_Rb_tree<unsigned long, unsigned long, std::_Identity<unsigned long>, std::less<unsigned long>, std::allocator<unsigned long> >::erase(unsigned long const&) [clone .isra.0]
     ||        + 0.20% librbd::io::AsyncOperation::finish_op()
     ||         + 0.20% pthread_mutex_lock
     |+ 0.20% librbd::io::ObjectWriteRequest<librbd::ImageCtx>::~ObjectWriteRequest()
     | + 0.20% ceph::buffer::v15_2_0::raw_posix_aligned::~raw_posix_aligned()
     |  + 0.20% mempool::pool_t::adjust_count(long, long)
     |   + 0.20% pthread_self
     + 0.40% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     | + 0.40% Context::complete(int)
     |  + 0.40% librbd::io::AioCompletion::complete_request(long)
     |   + 0.40% librbd::io::AioCompletion::complete()
     |    + 0.40% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     |     + 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |      + 0.40% librbd::io::AioCompletion::notify_callbacks_complete()
     |       + 0.40% Context::complete(int)
     |        + 0.20% librbd::io::ImageDispatchSpec::C_Dispatcher::complete(int)
     |        |+ 0.20% librbd::io::AioCompletion::put()
     |        + 0.20% LambdaContext<librbd::io::WriteBlockImageDispatch<librbd::ImageCtx>::process_io(unsigned long, librbd::io::DispatchResult*, Context**, Context*)::{lambda(int)#1}>::finish(int)
     + 0.20% pthread_mutex_lock
      + 0.20% __GI___lll_lock_wait

Thread 2999 (log) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::logging::Log::entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

Thread 3010 (ms_dispatch) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::DispatchThread::entry()
   + 100.00% DispatchQueue::entry()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3011 (ms_local) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::LocalDeliveryThread::entry()
   + 100.00% DispatchQueue::run_local_delivery()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3000 (msgr-worker-0) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 96.40% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 96.20% epoll_wait
     + 2.00% ProtocolV2::write_event()
     |+ 1.40% ProtocolV2::write_message(Message*, bool)
     ||+ 1.00% AsyncConnection::_try_send(bool)
     |||+ 1.00% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
     ||| + 1.00% sendmsg
     ||+ 0.20% ceph::msgr::v2::FrameAssembler::assemble_frame(ceph::msgr::v2::Tag, ceph::buffer::v15_2_0::list*, unsigned short const*, unsigned long)
     |||+ 0.20% ceph::msgr::v2::FrameAssembler::asm_crc_rev1(ceph::msgr::v2::preamble_block_t const&, ceph::buffer::v15_2_0::list*) const
     ||| + 0.20% ceph::buffer::v15_2_0::list::crc32c(unsigned int) const
     |||  + 0.20% crc32_iscsi_00
     ||+ 0.20% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::~MOSDOp()
     || + 0.20% __libc_free
     ||  + 0.20% _int_free
     |+ 0.40% Message::encode(unsigned long, int, bool)
     ||+ 0.40% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::encode_payload(unsigned long)
     || + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     || |+ 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     || | + 0.20% posix_memalign
     || |  + 0.20% malloc
     || |   + 0.20% _int_malloc
     || |    + 0.20% malloc_consolidate
     || + 0.20% ceph::buffer::v15_2_0::ptr::append(char const*, unsigned int)
     |+ 0.20% ProtocolV2::_get_next_outgoing()
     | + 0.20% __libc_free
     + 1.60% AsyncConnection::process()
      + 1.00% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
      |+ 0.60% ProtocolV2::handle_message()
      ||+ 0.60% DispatchQueue::fast_dispatch(Message*)
      || + 0.40% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
      || |+ 0.40% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
      || | + 0.40% Objecter::ms_dispatch(Message*)
      || |  + 0.40% Objecter::handle_osd_op_reply(MOSDOpReply*)
      || |   + 0.20% ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code>::destroy_defer(std::tuple<boost::system::error_code>&&) [clone .lto_priv.0]
      || |   |+ 0.20% boost::asio::detail::scheduler::post_immediate_completion(boost::asio::detail::scheduler_operation*, bool)
      || |   + 0.20% ceph::shunique_lock<std::shared_mutex>::unlock()
      || |    + 0.20% __pthread_rwlock_unlock
      || + 0.20% DispatchQueue::post_dispatch(boost::intrusive_ptr<Message> const&, unsigned long)
      ||  + 0.20% Throttle::put(long)
      ||   + 0.20% pthread_mutex_lock
      |+ 0.40% ProtocolV2::throttle_dispatch_queue()
      | + 0.40% Throttle::get_or_fail(long)
      |  + 0.20% __pthread_mutex_unlock
      |  + 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
      + 0.60% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
       + 0.60% AsyncConnection::read_until(unsigned int, char*)
        + 0.60% AsyncConnection::read_bulk(char*, unsigned int)
         + 0.60% PosixConnectedSocketImpl::read(char*, unsigned long)
          + 0.60% read

Thread 3001 (msgr-worker-1) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 95.20% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 95.20% epoll_wait
     + 2.40% AsyncConnection::process()
     |+ 1.40% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
     ||+ 1.20% ProtocolV2::handle_message()
     |||+ 1.00% DispatchQueue::fast_dispatch(Message*)
     ||||+ 1.00% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
     |||| + 1.00% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
     ||||  + 1.00% Objecter::ms_dispatch(Message*)
     ||||   + 1.00% Objecter::handle_osd_op_reply(MOSDOpReply*)
     ||||    + 0.20% std::__shared_mutex_pthread::lock_shared()
     ||||    |+ 0.20% __pthread_rwlock_rdlock
     ||||    + 0.20% std::unique_lock<std::recursive_mutex>::lock()
     ||||    |+ 0.20% pthread_mutex_lock
     ||||    + 0.20% Objecter::Op::~Op()
     ||||    + 0.20% Objecter::_finish_op(Objecter::Op*, int)
     ||||    + 0.20% ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code>::destroy_defer(std::tuple<boost::system::error_code>&&) [clone .lto_priv.0]
     ||||     + 0.20% pthread_cond_signal
     |||+ 0.20% decode_message(ceph::common::CephContext*, int, ceph_msg_header&, ceph_msg_footer&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, boost::intrusive_ptr<Connection>)
     ||| + 0.20% MOSDOpReply::decode_payload()
     |||  + 0.20% Message::decode_trace(ceph::buffer::v15_2_0::list::iterator_impl<true>&, bool)
     |||   + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, char*) [clone .localalias] [clone .lto_priv.0]
     |||    + 0.20% ceph::buffer::v15_2_0::ptr::copy_out(unsigned int, unsigned int, char*) const
     ||+ 0.20% ProtocolV2::_handle_read_frame_segment()
     || + 0.20% ceph::buffer::v15_2_0::ptr_node::create_hypercombined(ceph::unique_leakable_ptr<ceph::buffer::v15_2_0::raw>)
     ||  + 0.20% ceph::buffer::v15_2_0::ptr::ptr(ceph::unique_leakable_ptr<ceph::buffer::v15_2_0::raw>)
     |+ 1.00% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
     | + 1.00% AsyncConnection::read_until(unsigned int, char*)
     |  + 0.80% AsyncConnection::read_bulk(char*, unsigned int)
     |   + 0.80% PosixConnectedSocketImpl::read(char*, unsigned long)
     |    + 0.60% read
     + 2.40% ProtocolV2::write_event()
      + 1.40% ProtocolV2::write_message(Message*, bool)
      |+ 1.20% AsyncConnection::_try_send(bool)
      ||+ 1.20% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
      || + 0.80% sendmsg
      || + 0.40% ceph::buffer::v15_2_0::list::buffers_t::clear_and_dispose()
      ||  + 0.40% ceph::buffer::v15_2_0::raw_posix_aligned::~raw_posix_aligned()
      ||   + 0.40% __libc_free
      ||    + 0.20% _int_free
      |+ 0.20% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::~MOSDOp()
      + 0.60% Message::encode(unsigned long, int, bool)
      |+ 0.40% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::encode_payload(unsigned long)
      | + 0.20% Message::encode_trace(ceph::buffer::v15_2_0::list&, unsigned long) const
      | + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
      |  + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
      |   + 0.20% mempool::pool_t::adjust_count(long, long)
      + 0.40% ProtocolV2::_get_next_outgoing()
       + 0.20% __libc_free
       |+ 0.20% _int_free
       + 0.20% std::_Rb_tree<int, std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > >, std::_Select1st<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > >, std::less<int>, std::allocator<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > > >::erase(int const&) [clone .isra.0]
        + 0.20% std::_Rb_tree<int, std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > >, std::_Select1st<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > >, std::less<int>, std::allocator<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > > >::_M_erase(std::_Rb_tree_node<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > >*) [clone .isra.0]

Thread 3002 (msgr-worker-2) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 100.00% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
      + 100.00% epoll_wait

Thread 2998 (qemu-system-x86) - 500 samples
+ 100.00% __clone3
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% call_rcu_thread
    + 100.00% qemu_event_wait
     + 100.00% syscall

Thread 2974 (qemu-system-x86) - 500 samples
+ 97.40% _start
|+ 97.40% __libc_start_main
| + 97.40% __libc_start_call_main
|  + 97.40% qemu_default_main
|   + 97.40% qemu_main_loop
|    + 97.40% main_loop_wait
|     + 92.40% qemu_poll_ns
|     |+ 92.40% ppoll
|     + 1.80% g_main_context_dispatch
|     |+ 1.80% aio_ctx_dispatch
|     | + 1.80% aio_dispatch
|     |  + 1.20% aio_dispatch_handler
|     |  |+ 0.60% virtio_queue_notify_vq.part.0
|     |  ||+ 0.60% virtio_blk_handle_vq
|     |  || + 0.40% virtqueue_split_pop
|     |  || |+ 0.20% virtqueue_map_desc
|     |  || | + 0.20% address_space_map
|     |  || |  + 0.20% flatview_translate
|     |  || |   + 0.20% flatview_do_translate.isra.0
|     |  || |    + 0.20% address_space_translate_internal
|     |  || + 0.20% bdrv_io_plug
|     |  |+ 0.40% event_notifier_test_and_clear
|     |  ||+ 0.40% read
|     |  |+ 0.20% virtio_queue_host_notifier_read
|     |  | + 0.20% event_notifier_test_and_clear
|     |  |  + 0.20% read
|     |  + 0.40% aio_bh_poll
|     |  |+ 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
|     |  + 0.20% qemu_lockcnt_inc
|     |   + 0.20% qemu_lockcnt_cmpxchg_or_wait
|     + 1.00% g_main_context_prepare
|     |+ 0.60% io_watch_poll_prepare
|     ||+ 0.40% tcp_chr_read_poll
|     || + 0.40% chr_can_read
|     ||  + 0.40% virtio_serial_guest_ready
|     ||   + 0.20% virtio_queue_empty
|     |+ 0.20% clock_gettime
|     ||+ 0.20% ???
|     |+ 0.20% g_markup_parse_context_new
|     + 0.60% qemu_mutex_unlock_impl
|     |+ 0.60% __GI___pthread_mutex_unlock_usercnt
|     | + 0.60% __GI___lll_lock_wake
|     + 0.60% g_main_context_query
|     + 0.40% g_main_context_check
|     |+ 0.20% aio_ctx_check
|     + 0.20% qemu_clock_run_all_timers
|     |+ 0.20% timerlist_run_timers.part.0
|     | + 0.20% cpus_get_virtual_clock
|     + 0.20% g_main_context_default
|     + 0.20% timerlistgroup_deadline_ns
|      + 0.20% timerlist_deadline_ns
|       + 0.20% cpu_get_clock
|        + 0.20% cpu_get_clock_locked
|         + 0.20% clock_gettime
|          + 0.20% ???
+ 2.60% __push___start_context
 + 2.60% coroutine_trampoline
  + 0.80% blk_aio_flush_entry
  |+ 0.80% virtio_blk_flush_complete
  | + 0.40% event_notifier_set
  | |+ 0.40% __write
  | + 0.40% virtio_blk_req_complete
  |  + 0.40% virtqueue_push
  + 0.80% blk_aio_read_entry
  |+ 0.60% virtio_blk_rw_complete
  ||+ 0.40% event_notifier_set
  |||+ 0.40% __write
  ||+ 0.20% virtio_notify_irqfd
  || + 0.20% virtio_should_notify
  |+ 0.20% blk_co_do_preadv_part
  | + 0.20% bdrv_co_preadv_part
  |  + 0.20% bdrv_aligned_preadv
  |   + 0.20% bdrv_driver_preadv
  |    + 0.20% bdrv_co_preadv_part
  |     + 0.20% bdrv_aligned_preadv
  |      + 0.20% bdrv_driver_preadv
  |       + 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  |        + 0.20% rbd_aio_readv
  |         + 0.20% librbd::api::Io<librbd::ImageCtx>::aio_read(librbd::ImageCtx&, librbd::io::AioCompletion*, unsigned long, unsigned long, librbd::io::ReadResult&&, int, bool)
  |          + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  |           + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  |            + 0.20% librbd::io::QueueImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  |             + 0.20% void boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>::execute<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}> >(boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>&&) const [clone .lto_priv.0]
  |              + 0.20% pthread_cond_signal
  + 0.80% blk_aio_write_entry
   + 0.80% blk_co_do_pwritev_part
    + 0.80% bdrv_co_pwritev_part
     + 0.60% bdrv_aligned_pwritev
     |+ 0.60% bdrv_driver_pwritev
     | + 0.60% raw_co_pwritev
     |  + 0.60% bdrv_co_pwritev_part
     |   + 0.60% bdrv_aligned_pwritev
     |    + 0.60% bdrv_driver_pwritev
     |     + 0.60% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
     |      + 0.60% rbd_aio_writev
     |       + 0.40% librbd::api::Io<librbd::ImageCtx>::aio_write(librbd::ImageCtx&, librbd::io::AioCompletion*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, int, bool)
     |       |+ 0.20% clock_gettime
     |       |+ 0.20% std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()
     |       + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     |        + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |         + 0.20% librbd::io::QueueImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     |          + 0.20% void boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>::execute<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}> >(boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>&&) const [clone .lto_priv.0]
     |           + 0.20% pthread_cond_signal
     + 0.20% bdrv_pad_request

Thread 3014 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3013 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3012 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 3015 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 3006 (service) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::common::CephContextServiceThread::entry()
   + 100.00% pthread_cond_clockwait
    + 100.00% __futex_abstimed_wait_common

Thread 3016 (taskfin_librbd) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% Finisher::finisher_thread_entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

PWL Cache unwindpmp profiling.md

• VM configured with rbd_cache = false in both cases
• PWL enabled/disabled by setting rbd_plugins to pwl_cache/"" respectively
• PWL enabled/disabled status configured by output of rbd status libvirt-pool/pwl-test

1. Boot VM
2. Run perf test¹ in the guest without profiling in the host
3. Repeat twice:
   1. Run perf test in the guest
   2. Simultaneously run unwindpmp -n 900 -p <qemu pid> > ... on the host
   3. Ensure sampling starts after and finishes before test

500 samples was the maximum it was possible to collect while the PWL test was executing. It was possible to collect more when running without PWL cache, but I used 500 in both cases for consistency.

• nopwl-1.log: First run without PWL enabled
• nopwl-2.log: Second run without PWL enabled
• pwl-1.log: First run with PWL enabled
• pwl-2.log: Second run with PWL enabled

Footnotes

1. sudo podman run --rm --volume $(pwd):/var/lib/etcd:Z quay.io/openshift-scale/etcd-perf ↩

pwl-1.log

sample: 499

Thread 2807 (CPU 0/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2808 (CPU 1/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2809 (CPU 2/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2810 (CPU 3/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2806 (IO mon_iothread) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% iothread_run
    + 100.00% g_main_loop_run
     + 100.00% g_hmac_get_digest
      + 100.00% poll

Thread 2812 (bstore_aio) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% KernelDevice::AioCompletionThread::entry()
   + 100.00% KernelDevice::_aio_thread()
    + 98.80% aio_queue_t::get_next_completed(int, aio_t**, int)
    |+ 98.80% io_queue_run
    | + 98.80% syscall
    + 1.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_cache_cb(void*, void*)
     + 1.00% Context::complete(int)
     |+ 0.60% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&)::{lambda(int)#2}>::finish(int)
     ||+ 0.20% pthread_mutex_lock
     |||+ 0.20% __GI___lll_lock_wait
     ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::schedule_update_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, Context*) [clone .localalias] [clone .lto_priv.0]
     |||+ 0.20% std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_add_ref_copy()
     ||+ 0.20% clock_gettime
     || + 0.20% ???
     |+ 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_read_data_blocks(std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, Context*)::{lambda(int)#1}>::finish(int)
     |+ 0.20% Context::complete(int)
     | + 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops()::{lambda(int)#1}>::finish(int)
     |  + 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::retire_entries(unsigned long)::{lambda(int)#1}>::~LambdaContext()
     |   + 0.20% __libc_free
     + 0.20% ceph::buffer::v15_2_0::raw_posix_aligned::operator delete(void*)
      + 0.20% pthread_self

Thread 2813 (bstore_discard) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% KernelDevice::DiscardThread::entry()
   + 100.00% KernelDevice::_discard_thread()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2795 (ceph_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% ceph::timer<ceph::coarse_mono_clock>::timer_thread()
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2793 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 91.60% pthread_cond_wait
     |+ 91.20% __futex_abstimed_wait_common
     |+ 0.20% __GI___pthread_mutex_unlock_usercnt
     |+ 0.20% __pthread_mutex_cond_lock
     + 6.20% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 3.00% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 1.60% Context::complete(int)
     |||+ 1.20% librbd::io::AioCompletion::complete_request(long)
     ||||+ 1.00% librbd::io::AioCompletion::complete()
     |||| + 0.80% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     |||| |+ 0.80% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |||| | + 0.40% librbd::io::AioCompletion::notify_callbacks_complete()
     |||| | |+ 0.40% Context::complete(int)
     |||| | | + 0.40% Context::complete(int)
     |||| | |  + 0.20% Context::complete(int)
     |||| | |  |+ 0.20% librbd::io::ImageDispatchSpec::C_Dispatcher::complete(int)
     |||| | |  | + 0.20% librbd::io::AioCompletion::put()
     |||| | |  + 0.20% LambdaContext<librbd::io::QosImageDispatch<librbd::ImageCtx>::needs_throttle(bool, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > > const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)::{lambda(int)#1}>::finish(int)
     |||| | |   + 0.20% librbd::io::FlushTracker<librbd::ImageCtx>::finish_io(unsigned long)
     |||| | |    + 0.20% std::unique_lock<std::shared_mutex>::lock()
     |||| | |     + 0.20% __pthread_rwlock_wrlock
     |||| | + 0.40% aio_bh_schedule_oneshot_full
     |||| |  + 0.40% event_notifier_set
     |||| |   + 0.40% __write
     |||| + 0.20% void librbd::io::AioCompletion::callback_adapter<Context, &Context::complete>(void*, void*)
     ||||  + 0.20% Context::complete(int)
     ||||   + 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::construct_flush_entry(std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, bool)::{lambda(int)#1}>::finish(int)
     ||||    + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
     ||||     + 0.20% std::condition_variable::notify_one()
     ||||      + 0.20% pthread_cond_signal
     |||+ 0.40% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool)::{lambda(int)#1}::operator()(int) const::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}::operator()(librbd::cache::pwl::GuardedRequestFunctionContext&) const::{lambda(int)#1}>::finish(int)
     ||| + 0.40% librbd::cache::pwl::ssd::WriteLogEntry::writeback_bl(librbd::cache::ImageWritebackInterface&, Context*, ceph::buffer::v15_2_0::list&&)
     |||  + 0.40% librbd::cache::ImageWriteback<librbd::ImageCtx>::aio_write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*)
     |||   + 0.20% std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()
     |||   + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     |||    + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||+ 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     |||+ 1.00% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||| + 0.80% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     ||| |+ 0.80% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*) [clone .localalias]
     ||| | + 0.60% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
     ||| | |+ 0.60% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::alloc_and_dispatch_io_req(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| | | + 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::alloc_resources(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| | |  + 0.60% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::check_allocation(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, unsigned long, unsigned long, unsigned long, unsigned int, unsigned int, unsigned int) [clone .localalias] [clone .lto_priv.0]
     ||| | |   + 0.60% librbd::cache::pwl::ImageCacheState<librbd::ImageCtx>::write_image_cache_state(std::unique_lock<std::mutex>&, Context*) [clone .localalias] [clone .lto_priv.0]
     ||| | |    + 0.20% json_spirit::write(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, unsigned int)
     ||| | |    |+ 0.20% json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >::String_type json_spirit::write_string<json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, unsigned int)
     ||| | |    | + 0.20% json_spirit::Generator<json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> > >::output(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&)
     ||| | |    |  + 0.20% __libc_free
     ||| | |    + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string<std::allocator<char> >(char const*, std::allocator<char> const&)
     ||| | |    + 0.20% librbd::operation::MetadataSetRequest<librbd::ImageCtx>::send_op()
     ||| | |     + 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectWriteOperation*)
     ||| | |      + 0.20% librados::IoCtxImpl::aio_operate(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, SnapContext const&, int, blkin_trace_info const*) [clone .isra.0]
     ||| | |       + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||| | |        + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||| | |         + 0.20% Objecter::_send_op(Objecter::Op*)
     ||| | |          + 0.20% AsyncConnection::send_message(Message*)
     ||| | |           + 0.20% ProtocolV2::send_message(Message*)
     ||| | |            + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     ||| | |             + 0.20% EventCenter::wakeup()
     ||| | |              + 0.20% __write
     ||| | + 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     ||| + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |||  + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::read(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list*, int, Context*) [clone .localalias] [clone .lto_priv.0]
     |||   + 0.20% KernelDevice::aio_submit(IOContext*)
     |||    + 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
     |||     + 0.20% io_submit
     |||      + 0.20% syscall
     ||+ 0.40% librbd::util::detail::C_StateCallbackAdapter<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_apply, true>::complete(int)
     || + 0.40% librbd::image::RefreshRequest<librbd::ImageCtx>::~RefreshRequest() [clone .lto_priv.0]
     ||  + 0.40% std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >*) [clone .isra.0]
     |+ 2.80% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librados::CB_AioComplete>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 1.00% void librbd::util::detail::rados_state_callback<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_mutable_metadata, true>(void*, void*)
     |||+ 1.00% librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_mutable_metadata(int*)
     ||| + 1.00% librbd::image::RefreshRequest<librbd::ImageCtx>::send_v2_get_parent()
     |||  + 0.80% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |||  |+ 0.60% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |||  | + 0.60% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |||  |  + 0.40% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |||  |  |+ 0.40% Objecter::_send_op(Objecter::Op*)
     |||  |  | + 0.20% AsyncConnection::send_message(Message*)
     |||  |  | |+ 0.20% ProtocolV2::send_message(Message*)
     |||  |  | | + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     |||  |  | |  + 0.20% EventCenter::wakeup()
     |||  |  | |   + 0.20% __write
     |||  |  | + 0.20% boost::move_detail::disable_if_or<void, boost::container::dtl::is_version<boost::container::real_allocator<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >::type, 0u>, boost::move_detail::is_different<boost::container::real_allocator<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >::type, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >, boost::move_detail::bool_<false>, boost::move_detail::bool_<false> >::type boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void>::priv_copy_assign<boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >(boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void> const&)
     |||  |  |  + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) [clone .lto_priv.0]
     |||  |  + 0.20% __pthread_rwlock_rdlock
     |||  + 0.20% librbd::cls_client::parent_overlap_get_start(librados::v14_2_0::ObjectReadOperation*, snapid_t)
     |||   + 0.20% librados::v14_2_0::ObjectOperation::exec(char const*, char const*, ceph::buffer::v15_2_0::list&)
     |||    + 0.20% ceph::buffer::v15_2_0::ptr::append(char const*, unsigned int)
     ||+ 1.00% void librbd::util::detail::rados_state_callback<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent, true>(void*, void*)
     |||+ 1.00% librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent(int*)
     ||| + 1.00% librbd::image::GetMetadataRequest<librbd::ImageCtx>::metadata_list()
     |||  + 0.60% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |||  |+ 0.60% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |||  | + 0.40% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |||  | |+ 0.40% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |||  | | + 0.40% Objecter::_send_op(Objecter::Op*)
     |||  | |  + 0.20% __libc_free
     |||  | |  |+ 0.20% _int_free
     |||  | |  + 0.20% hobject_t::hobject_t(hobject_t const&)
     |||  | |   + 0.20% operator new(unsigned long)
     |||  | |    + 0.20% malloc
     |||  | |     + 0.20% _int_malloc
     |||  | + 0.20% Objecter::prepare_read_op(object_t const&, object_locator_t const&, ObjectOperation&, snapid_t, ceph::buffer::v15_2_0::list*, int, Context*, unsigned long*, int*, unsigned long, ZTracer::Trace*) [clone .constprop.0] [clone .isra.0]
     |||  |  + 0.20% operator new(unsigned long)
     |||  |   + 0.20% malloc
     |||  + 0.20% operator delete(void*, unsigned long)
     ||+ 0.20% librbd::operation::Request<librbd::ImageCtx>::finish_and_destroy(int)
     |||+ 0.20% librbd::ImageWatcher<librbd::ImageCtx>::send_notify(librbd::watch_notify::Payload*, Context*)
     ||| + 0.20% librbd::watch_notify::NotifyMessage::encode(ceph::buffer::v15_2_0::list&) const
     |||  + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     |||   + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     |||    + 0.20% posix_memalign
     |||     + 0.20% malloc
     |||      + 0.20% _int_malloc
     |||       + 0.20% malloc_consolidate
     ||+ 0.20% void librbd::util::detail::rados_callback<librbd::AsyncRequest<librbd::ImageCtx> >(void*, void*)
     |||+ 0.20% _rados_aio_get_return_value
     ||| + 0.20% pthread_mutex_lock
     ||+ 0.20% librbd::operation::MetadataSetRequest<librbd::ImageCtx>::~MetadataSetRequest()
     |||+ 0.20% __libc_free
     ||| + 0.20% _int_free
     ||+ 0.20% librbd::image::GetMetadataRequest<librbd::ImageCtx>::handle_metadata_list(int)
     || + 0.20% std::enable_if<denc_traits<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >, void>::supported&&denc_traits<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >, void>::need_contiguous, void>::type ceph::decode<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >, denc_traits<std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >, void> >(std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, ceph::buffer::v15_2_0::list::iterator_impl<true>&)
     ||  + 0.20% operator new(unsigned long)
     ||   + 0.20% malloc
     ||    + 0.20% _int_malloc
     |+ 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<CB_DoWatchNotify>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     | + 0.40% WatchInfo::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |  + 0.40% librbd::Watcher::WatchCtx::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |   + 0.20% librbd::ImageWatcher<librbd::ImageCtx>::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |   |+ 0.20% librbd::ImageWatcher<librbd::ImageCtx>::process_payload(unsigned long, unsigned long, librbd::watch_notify::Payload*)
     |   | + 0.20% librbd::ImageWatcher<librbd::ImageCtx>::handle_payload(librbd::watch_notify::HeaderUpdatePayload const&, librbd::Watcher::C_NotifyAck*)
     |   |  + 0.20% librbd::ImageWatcher<librbd::ImageCtx>::C_ResponseMessage::finish(int)
     |   |   + 0.20% librbd::watch_notify::ResponseMessage::encode(ceph::buffer::v15_2_0::list&) const
     |   |    + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     |   |     + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     |   |      + 0.20% posix_memalign
     |   |       + 0.20% malloc
     |   |        + 0.20% __lll_lock_wake_private
     |   + 0.20% librbd::Watcher::notifications_blocked() const
     |    + 0.20% std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long)
     |     + 0.20% std::ostream::sentry::sentry(std::ostream&)
     + 0.80% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.80% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 0.60% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |  + 0.40% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |  |+ 0.40% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::read(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list*, int, Context*) [clone .localalias] [clone .lto_priv.0]
     |  | + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::collect_read_extents(unsigned long, librbd::cache::pwl::LogMapEntry<librbd::cache::pwl::GenericWriteLogEntry>, std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, unsigned long, std::pair<unsigned long, unsigned long>, librbd::cache::pwl::C_ReadRequest*) [clone .localalias] [clone .lto_priv.0]
     |  |  + 0.40% ceph::logging::Log::submit_entry(ceph::logging::Entry&&)
     |  |   + 0.40% std::condition_variable::notify_all()
     |  |    + 0.40% pthread_cond_broadcast
     |  + 0.20% librbd::io::QosImageDispatch<librbd::ImageCtx>::needs_throttle(bool, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > > const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     + 0.60% __GI___pthread_mutex_unlock_usercnt
     |+ 0.60% __GI___lll_lock_wake
     + 0.40% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.20% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::post_write_object_map_update()
     ||+ 0.20% __pthread_rwlock_rdlock
     |+ 0.20% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
     | + 0.20% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
     |  + 0.20% Context::complete(int)
     |   + 0.20% LambdaContext<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::register_in_flight_request(unsigned long, utime_t const&, Context**)::{lambda(int)#1}>::finish(int)
     |    + 0.20% Context::complete(int)
     |     + 0.20% __libc_free
     |      + 0.20% _int_free
     + 0.20% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librados::(anonymous namespace)::CB_notify_Finish, std::tuple<boost::system::error_code, ceph::buffer::v15_2_0::list> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librados::(anonymous namespace)::CB_notify_Finish, void, boost::system::error_code, ceph::buffer::v15_2_0::list> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
      + 0.20% librados::(anonymous namespace)::CB_notify_Finish::operator()(boost::system::error_code, ceph::buffer::v15_2_0::list&&) [clone .lto_priv.0]
       + 0.20% librados::(anonymous namespace)::C_aio_notify_Complete::complete(int) [clone .lto_priv.0]
        + 0.20% pthread_cond_signal

Thread 2794 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 98.00% pthread_cond_wait
     |+ 97.80% __futex_abstimed_wait_common
     + 1.60% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 1.00% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 0.60% Context::complete(int)
     |||+ 0.40% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool)::{lambda(int)#1}::operator()(int) const::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}::operator()(librbd::cache::pwl::GuardedRequestFunctionContext&) const::{lambda(int)#1}>::finish(int)
     ||||+ 0.40% librbd::cache::pwl::ssd::WriteLogEntry::writeback_bl(librbd::cache::ImageWritebackInterface&, Context*, ceph::buffer::v15_2_0::list&&)
     |||| + 0.40% librbd::cache::ImageWriteback<librbd::ImageCtx>::aio_write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*)
     ||||  + 0.20% std::_Sp_locker::_Sp_locker(void const*)
     ||||  |+ 0.20% std::bad_weak_ptr::~bad_weak_ptr()
     ||||  + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     ||||   + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||||    + 0.20% librbd::ImageCtx::get_effective_image_size(unsigned long) const
     ||||     + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::remap_extents(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&, librbd::io::ImageExtentsMapType)
     ||||      + 0.20% std::_Rb_tree_decrement(std::_Rb_tree_node_base const*)
     |||+ 0.20% librbd::io::AioCompletion::complete_request(long)
     ||| + 0.20% librbd::io::AioCompletion::complete()
     |||  + 0.20% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     |||   + 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |||    + 0.20% aio_bh_schedule_oneshot_full
     |||     + 0.20% event_notifier_set
     |||      + 0.20% __write
     ||+ 0.40% librbd::util::detail::C_StateCallbackAdapter<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_apply, true>::complete(int)
     || + 0.20% librbd::image::RefreshRequest<librbd::ImageCtx>::~RefreshRequest() [clone .lto_priv.0]
     ||  + 0.20% std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >*) [clone .isra.0]
     |+ 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<CB_DoWatchNotify>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     | + 0.40% WatchInfo::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |  + 0.40% librbd::Watcher::WatchCtx::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |   + 0.40% librbd::Watcher::notifications_blocked() const
     |    + 0.20% ceph::logging::log_clock::coarse_now()
     |    |+ 0.20% clock_gettime
     |    | + 0.20% ???
     |    + 0.20% std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long)
     |     + 0.20% std::ostream::sentry::sentry(std::ostream&)
     + 0.20% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |  + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::read(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list*, int, Context*) [clone .localalias] [clone .lto_priv.0]
     |    + 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     + 0.20% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
      + 0.20% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
       + 0.20% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
        + 0.20% Context::complete(int)
         + 0.20% LambdaContext<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::register_in_flight_request(unsigned long, utime_t const&, Context**)::{lambda(int)#1}>::finish(int)
          + 0.20% Context::complete(int)
           + 0.20% librbd::io::AioCompletion::complete_request(long)
            + 0.20% librbd::io::AioCompletion::complete()
             + 0.20% void librbd::io::AioCompletion::callback_adapter<Context, &Context::complete>(void*, void*)
              + 0.20% Context::complete(int)
               + 0.20% librbd::cache::ImageWriteback<librbd::ImageCtx>::aio_flush(librbd::io::FlushSource, Context*)
                + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
                 + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
                  + 0.20% librbd::io::ImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
                   + 0.20% librbd::io::ImageRequest<librbd::ImageCtx>::aio_flush(librbd::ImageCtx*, librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&)
                    + 0.20% librbd::io::ImageFlushRequest<librbd::ImageCtx>::send_request()
                     + 0.20% _init

Thread 2785 (log) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::logging::Log::entry()
   + 99.80% std::condition_variable::wait(std::unique_lock<std::mutex>&)
   |+ 99.60% pthread_cond_wait
   | + 99.40% __futex_abstimed_wait_common
   | + 0.20% __condvar_dec_grefs
   + 0.20% __pthread_mutex_unlock

Thread 2796 (ms_dispatch) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::DispatchThread::entry()
   + 100.00% DispatchQueue::entry()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2797 (ms_local) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::LocalDeliveryThread::entry()
   + 100.00% DispatchQueue::run_local_delivery()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2786 (msgr-worker-0) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 97.40% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 97.40% epoll_wait
     + 1.60% AsyncConnection::process()
     |+ 1.00% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
     ||+ 0.80% ProtocolV2::handle_message()
     |||+ 0.20% DispatchQueue::fast_dispatch(Message*)
     ||||+ 0.20% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
     |||| + 0.20% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
     ||||  + 0.20% Objecter::ms_dispatch(Message*)
     ||||   + 0.20% Objecter::handle_osd_op_reply(MOSDOpReply*)
     ||||    + 0.20% Context::complete(int)
     ||||     + 0.20% librados::IoCtxImpl::C_aio_Complete::finish(int)
     ||||      + 0.20% auto boost::asio::defer<boost::asio::io_context::strand, librados::CB_AioComplete>(boost::asio::io_context::strand const&, librados::CB_AioComplete&&, boost::asio::constraint<boost::asio::execution::is_executor<boost::asio::io_context::strand>::value||boost::asio::is_executor<boost::asio::io_context::strand>::value, int>::type) [clone .constprop.0]
     ||||       + 0.20% pthread_cond_signal
     |||+ 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     |||+ 0.20% ceph::common::RefCountedObject::_get() const
     ||+ 0.20% ProtocolV2::read_frame_segment()
     || + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
     ||  + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
     ||   + 0.20% posix_memalign
     ||    + 0.20% malloc
     |+ 0.60% AsyncConnection::read(unsigned int, char*, std::function<void (char*, long)>)
     | + 0.40% AsyncConnection::read_until(unsigned int, char*)
     |  + 0.40% AsyncConnection::read_bulk(char*, unsigned int)
     |   + 0.20% PosixConnectedSocketImpl::read(char*, unsigned long)
     |    + 0.20% read
     + 1.00% ProtocolV2::write_event()
      + 1.00% ProtocolV2::write_message(Message*, bool)
       + 0.80% AsyncConnection::_try_send(bool)
       |+ 0.80% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
       | + 0.60% sendmsg
       | + 0.20% ceph::buffer::v15_2_0::list::buffers_t::clear_and_dispose()
       |  + 0.20% __libc_free
       + 0.20% ceph::buffer::v15_2_0::ptr_node::cloner::operator()(ceph::buffer::v15_2_0::ptr_node const&)
        + 0.20% ceph::buffer::v15_2_0::ptr::ptr(ceph::buffer::v15_2_0::ptr const&)

Thread 2787 (msgr-worker-1) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 99.40% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 99.40% epoll_wait
     + 0.40% AsyncConnection::process()
     |+ 0.40% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
     | + 0.20% ProtocolV2::handle_message()
     | |+ 0.20% DispatchQueue::fast_dispatch(Message*)
     | | + 0.20% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
     | |  + 0.20% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
     | |   + 0.20% Objecter::ms_dispatch(Message*)
     | |    + 0.20% Objecter::handle_osd_op_reply(MOSDOpReply*)
     | |     + 0.20% Objecter::Op::~Op()
     | |      + 0.20% boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void>::~vector() [clone .lto_priv.0]
     | |       + 0.20% __libc_free
     | |        + 0.20% _int_free
     | + 0.20% ProtocolV2::throttle_dispatch_queue()
     |  + 0.20% Throttle::get_or_fail(long)
     |   + 0.20% ceph::common::PerfCounters::set(int, unsigned long)
     + 0.20% ProtocolV2::write_event()
      + 0.20% ProtocolV2::write_message(Message*, bool)
       + 0.20% ceph::msgr::v2::FrameAssembler::assemble_frame(ceph::msgr::v2::Tag, ceph::buffer::v15_2_0::list*, unsigned short const*, unsigned long)
        + 0.20% ceph::msgr::v2::FrameAssembler::asm_crc_rev1(ceph::msgr::v2::preamble_block_t const&, ceph::buffer::v15_2_0::list*) const
         + 0.20% ceph::buffer::v15_2_0::list::reserve(unsigned long)
          + 0.20% ceph::buffer::v15_2_0::create_small_page_aligned(unsigned int)
           + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
            + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
             + 0.20% posix_memalign
              + 0.20% _mid_memalign.isra.0
               + 0.20% _int_memalign
                + 0.20% _int_malloc

Thread 2788 (msgr-worker-2) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 100.00% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
      + 100.00% epoll_wait

Thread 2762 (qemu-system-x86) - 500 samples
+ 98.20% _start
|+ 98.20% __libc_start_main
| + 98.20% __libc_start_call_main
|  + 98.20% qemu_default_main
|   + 98.20% qemu_main_loop
|    + 98.20% main_loop_wait
|     + 97.60% qemu_poll_ns
|     |+ 97.60% ppoll
|     + 0.40% g_main_context_dispatch
|     |+ 0.20% aio_ctx_dispatch
|     ||+ 0.20% aio_dispatch
|     || + 0.20% aio_dispatch_handler
|     ||  + 0.20% event_notifier_test_and_clear
|     ||   + 0.20% read
|     |+ 0.20% g_ptr_array_set_size
|     + 0.20% g_main_context_prepare
|      + 0.20% g_get_monotonic_time
+ 1.80% __push___start_context
 + 1.80% coroutine_trampoline
  + 1.60% blk_aio_flush_entry
  |+ 1.40% bdrv_co_flush
  ||+ 1.40% bdrv_co_flush
  || + 1.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  || |+ 1.20% rbd_aio_flush
  || | + 1.20% librbd::api::Io<librbd::ImageCtx>::aio_flush(librbd::ImageCtx&, librbd::io::AioCompletion*, bool)
  || |  + 1.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  || |   + 1.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  || |    + 1.20% librbd::io::QueueImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  || |     + 1.20% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
  || |      + 1.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  || |       + 1.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  || |        + 1.20% librbd::io::QosImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  || |         + 1.20% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
  || |          + 1.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  || |           + 1.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  || |            + 1.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
  || |             + 1.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
  || |              + 1.20% boost::detail::function::void_function_obj_invoker1<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::flush(librbd::io::FlushSource, Context*)::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}, void, librbd::cache::pwl::GuardedRequestFunctionContext&>::invoke(boost::detail::function::function_buffer&, librbd::cache::pwl::GuardedRequestFunctionContext&)
  || |               + 1.20% librbd::cache::pwl::DeferredContexts::~DeferredContexts()
  || |                + 1.00% Context::complete(int)
  || |                |+ 1.00% librbd::cache::pwl::SyncPoint::persist_gather_activate()
  || |                | + 1.00% Context::complete(int)
  || |                |  + 1.00% librbd::cache::pwl::C_FlushRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::dispatch() [clone .localalias] [clone .lto_priv.0]
  || |                |   + 1.00% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::schedule_append(std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
  || |                |    + 1.00% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::schedule_append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
  || |                |     + 1.00% librbd::cache::pwl::SyncPointLogOperation::appending()
  || |                |      + 1.00% librbd::cache::pwl::SyncPointLogOperation::append_sync_point()
  || |                |       + 1.00% pthread_mutex_lock
  || |                |        + 1.00% __GI___lll_lock_wait
  || |                + 0.20% __libc_free
  || + 0.20% qemu_co_mutex_unlock
  |+ 0.20% virtio_blk_flush_complete
  | + 0.20% virtio_blk_req_complete
  |  + 0.20% virtqueue_push
  |   + 0.20% virtqueue_fill
  |    + 0.20% virtqueue_unmap_sg.isra.0
  |     + 0.20% address_space_unmap
  |      + 0.20% memory_region_from_host
  + 0.20% blk_aio_write_entry
   + 0.20% blk_co_do_pwritev_part
    + 0.20% bdrv_co_pwritev_part
     + 0.20% bdrv_aligned_pwritev
      + 0.20% bdrv_driver_pwritev
       + 0.20% raw_co_pwritev
        + 0.20% bdrv_co_pwritev_part
         + 0.20% bdrv_aligned_pwritev
          + 0.20% bdrv_driver_pwritev
           + 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
            + 0.20% rbd_aio_writev
             + 0.20% (anonymous namespace)::iovec_to_bufferlist(librbd::ImageCtx*, iovec const*, int, librbd::io::AioCompletion*) [clone .lto_priv.0]
              + 0.20% (anonymous namespace)::create_write_raw(librbd::ImageCtx*, char const*, unsigned long, librbd::io::AioCompletion*) [clone .lto_priv.0]
               + 0.20% ceph::buffer::v15_2_0::copy(char const*, unsigned int)
                + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
                 + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
                  + 0.20% ceph::buffer::v15_2_0::raw_combined::create(unsigned int, unsigned int, int)
                   + 0.20% posix_memalign
                    + 0.20% malloc
                     + 0.20% _int_malloc
                      + 0.20% malloc_consolidate

Thread 2784 (qemu-system-x86) - 500 samples
+ 100.00% __clone3
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% call_rcu_thread
    + 100.00% qemu_event_wait
     + 100.00% syscall

Thread 2798 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2799 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2801 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2800 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2792 (service) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::common::CephContextServiceThread::entry()
   + 100.00% pthread_cond_clockwait
    + 100.00% __futex_abstimed_wait_common

Thread 2802 (taskfin_librbd) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% Finisher::finisher_thread_entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

Thread 2815 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 96.60% pthread_cond_clockwait
    |+ 96.40% __futex_abstimed_wait_common
    |+ 0.20% __pthread_mutex_cond_lock
    | + 0.20% __GI___lll_lock_wait
    + 2.40% Context::complete(int)
    |+ 1.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
    ||+ 1.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
    || + 0.80% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
    || |+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool) [clone .localalias] [clone .lto_priv.0]
    || ||+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_read_data_blocks(std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, Context*) [clone .localalias] [clone .lto_priv.0]
    || || + 0.40% KernelDevice::aio_read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*)
    || ||  + 0.20% KernelDevice::create_custom_aligned(unsigned long, IOContext*) const
    || ||   + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
    || ||    + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
    || ||     + 0.20% posix_memalign
    || ||      + 0.20% _mid_memalign.isra.0
    || ||       + 0.20% _int_memalign
    || ||        + 0.20% _int_free
    || ||         + 0.20% unlink_chunk.isra.0
    || |+ 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::can_flush_entry(std::shared_ptr<librbd::cache::pwl::GenericLogEntry>) [clone .localalias]
    || ||+ 0.20% librbd::cache::pwl::GenericWriteLogEntry::can_writeback() const
    || |+ 0.20% librbd::cache::pwl::ImageCacheState<librbd::ImageCtx>::write_image_cache_state(std::unique_lock<std::mutex>&, Context*) [clone .localalias] [clone .lto_priv.0]
    || | + 0.20% std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >::operator[](std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&)
    || |  + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::compare(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const [clone .isra.0]
    || + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::retire_entries(unsigned long) [clone .localalias] [clone .lto_priv.0]
    ||  + 0.20% librbd::cache::pwl::LogMap<librbd::cache::pwl::GenericWriteLogEntry>::remove_log_entry(std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>) [clone .localalias] [clone .lto_priv.0]
    ||  |+ 0.20% librbd::cache::pwl::LogMap<librbd::cache::pwl::GenericWriteLogEntry>::remove_log_entry_locked(std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>) [clone .localalias]
    ||  | + 0.20% librbd::cache::pwl::LogMap<librbd::cache::pwl::GenericWriteLogEntry>::find_map_entries_locked[abi:cxx11](librbd::BlockExtent const&) [clone .localalias]
    ||  + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
    ||   + 0.20% std::condition_variable::notify_one()
    ||    + 0.20% pthread_cond_signal
    |+ 0.80% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    |||+ 0.20% KernelDevice::aio_submit(IOContext*)
    ||||+ 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    |||| + 0.20% io_submit
    ||||  + 0.20% syscall
    |||+ 0.20% __memset_evex_unaligned_erms
    |||+ 0.20% ceph::buffer::v15_2_0::list::append_zero(unsigned int)
    ||| + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
    |||  + 0.20% posix_memalign
    |||   + 0.20% malloc
    |||    + 0.20% _int_malloc
    |||     + 0.20% malloc_consolidate
    ||+ 0.20% std::recursive_mutex::lock()
    || + 0.20% pthread_mutex_lock
    ||  + 0.20% __GI___lll_lock_wait
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::handle_flushed_sync_point(std::shared_ptr<librbd::cache::pwl::SyncPointLogEntry>)::{lambda(int)#1}>::finish(int)
    ||+ 0.20% pthread_mutex_lock
    |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    | + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::dispatch_deferred_writes() [clone .localalias] [clone .lto_priv.0]
    |  + 0.20% pthread_mutex_lock
    |   + 0.20% __GI___lll_lock_wait
    + 0.60% std::unique_lock<std::mutex>::lock()
    |+ 0.60% pthread_mutex_lock
    | + 0.60% __GI___lll_lock_wait
    + 0.40% __GI___pthread_mutex_unlock_usercnt
     + 0.20% __GI___lll_lock_wake

Thread 2816 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 97.40% pthread_cond_clockwait
    |+ 96.80% __futex_abstimed_wait_common
    |+ 0.40% __pthread_mutex_cond_lock
    ||+ 0.20% __GI___lll_lock_wait
    |+ 0.20% __condvar_dec_grefs
    + 1.40% Context::complete(int)
    |+ 1.00% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    ||+ 0.80% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::dispatch_deferred_writes() [clone .localalias] [clone .lto_priv.0]
    |||+ 0.80% pthread_mutex_lock
    ||| + 0.80% __GI___lll_lock_wait
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
    || + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
    ||  + 0.20% std::recursive_mutex::lock()
    ||   + 0.20% pthread_mutex_lock
    ||    + 0.20% __GI___lll_lock_wait
    |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    || + 0.20% KernelDevice::aio_write(unsigned long, ceph::buffer::v15_2_0::list&, IOContext*, bool, int)
    ||  + 0.20% ceph::buffer::v15_2_0::list::rebuild_aligned_size_and_memory(unsigned int, unsigned int, unsigned int)
    ||   + 0.20% ceph::buffer::v15_2_0::list::rebuild(std::unique_ptr<ceph::buffer::v15_2_0::ptr_node, ceph::buffer::v15_2_0::ptr_node::disposer>)
    ||    + 0.20% ceph::buffer::v15_2_0::raw_combined::~raw_combined()
    ||     + 0.20% mempool::get_pool(mempool::pool_index_t)
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::handle_flushed_sync_point(std::shared_ptr<librbd::cache::pwl::SyncPointLogEntry>)::{lambda(int)#1}>::finish(int)
    | + 0.20% pthread_mutex_lock
    |  + 0.20% __GI___lll_lock_wait
    + 0.80% std::unique_lock<std::mutex>::lock()
    |+ 0.80% pthread_mutex_lock
    | + 0.80% __GI___lll_lock_wait
    + 0.20% ThreadPool::join_old_threads()
    + 0.20% ThreadPool::PointerWQ<Context>::_void_dequeue()

Thread 2814 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 97.80% pthread_cond_clockwait
    |+ 97.80% __futex_abstimed_wait_common
    + 1.80% Context::complete(int)
    |+ 0.80% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    ||+ 0.80% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
    || + 0.80% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
    ||  + 0.60% KernelDevice::aio_submit(IOContext*)
    ||  |+ 0.60% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    ||  | + 0.60% io_submit
    ||  |  + 0.60% syscall
    ||  + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::write_log_entries(std::vector<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*, unsigned long*) [clone .localalias] [clone .lto_priv.0]
    ||   + 0.20% ceph::buffer::v15_2_0::list::append_zero(unsigned int)
    ||    + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
    ||     + 0.20% posix_memalign
    ||      + 0.20% malloc
    ||       + 0.20% _int_malloc
    ||        + 0.20% malloc_consolidate
    |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    || + 0.40% KernelDevice::aio_submit(IOContext*)
    || |+ 0.40% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    || | + 0.40% io_submit
    || |  + 0.40% syscall
    || + 0.20% ceph::buffer::v15_2_0::list::append_zero(unsigned int)
    ||  + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
    ||   + 0.20% posix_memalign
    ||    + 0.20% malloc
    ||     + 0.20% _int_malloc
    ||      + 0.20% malloc_consolidate
    |+ 0.40% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
    | + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
    |  + 0.40% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
    |   + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool) [clone .localalias] [clone .lto_priv.0]
    |    + 0.40% KernelDevice::aio_submit(IOContext*)
    |     + 0.40% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    |      + 0.40% io_submit
    |       + 0.40% syscall
    + 0.20% std::chrono::_V2::steady_clock::now()
    |+ 0.20% std::__cxx11::basic_stringstream<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >::str() const &
    + 0.20% __GI___pthread_mutex_unlock_usercnt
     + 0.20% __GI___lll_lock_wake

Thread 2817 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 98.40% pthread_cond_clockwait
    |+ 98.00% __futex_abstimed_wait_common
    |+ 0.40% __pthread_mutex_cond_lock
    | + 0.40% __GI___lll_lock_wait
    + 1.40% Context::complete(int)
    |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    ||+ 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::dispatch_deferred_writes() [clone .localalias] [clone .lto_priv.0]
    |||+ 0.20% pthread_mutex_lock
    ||| + 0.20% __GI___lll_lock_wait
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
    |||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
    ||| + 0.20% KernelDevice::aio_submit(IOContext*)
    |||  + 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    |||   + 0.20% io_submit
    |||    + 0.20% syscall
    ||+ 0.20% std::__cxx11::_List_base<std::shared_ptr<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::WriteLogPoolRootUpdate>, std::allocator<std::shared_ptr<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::WriteLogPoolRootUpdate> > >::_M_clear()
    |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.40% std::recursive_mutex::lock()
    |||+ 0.40% pthread_mutex_lock
    ||| + 0.40% __GI___lll_lock_wait
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    || + 0.20% KernelDevice::aio_submit(IOContext*)
    ||  + 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    ||   + 0.20% io_submit
    ||    + 0.20% syscall
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
    | + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
    |  + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
    |   + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool) [clone .localalias] [clone .lto_priv.0]
    |    + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_read_data_blocks(std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, Context*) [clone .localalias] [clone .lto_priv.0]
    |     + 0.20% KernelDevice::aio_read(unsigned long, unsigned long, ceph::buffer::v15_2_0::list*, IOContext*)
    |      + 0.20% KernelDevice::create_custom_aligned(unsigned long, IOContext*) const
    |       + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
    |        + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
    |         + 0.20% posix_memalign
    |          + 0.20% _mid_memalign.isra.0
    + 0.20% std::unique_lock<std::mutex>::lock()
     + 0.20% pthread_mutex_lock
      + 0.20% __GI___lll_lock_wait

pwl-2.log

sample: 499

Thread 2807 (CPU 0/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2808 (CPU 1/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2809 (CPU 2/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2810 (CPU 3/KVM) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% kvm_vcpu_thread_fn
    + 100.00% kvm_cpu_exec
     + 100.00% kvm_vcpu_ioctl
      + 100.00% ioctl

Thread 2806 (IO mon_iothread) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% iothread_run
    + 100.00% g_main_loop_run
     + 100.00% g_hmac_get_digest
      + 100.00% poll

Thread 2812 (bstore_aio) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% KernelDevice::AioCompletionThread::entry()
   + 100.00% KernelDevice::_aio_thread()
    + 98.20% aio_queue_t::get_next_completed(int, aio_t**, int)
    |+ 98.20% io_queue_run
    | + 98.20% syscall
    + 1.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_cache_cb(void*, void*)
    |+ 1.40% Context::complete(int)
    ||+ 1.00% Context::complete(int)
    |||+ 1.00% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops()::{lambda(int)#1}>::finish(int)
    ||| + 0.60% Context::complete(int)
    ||| |+ 0.60% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&)::{lambda(int)#1}>::finish(int)
    ||| | + 0.60% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::complete_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&&, int) [clone .localalias] [clone .lto_priv.0]
    ||| |  + 0.60% librbd::cache::pwl::SyncPointLogOperation::complete(int)
    ||| |   + 0.40% Context::complete(int)
    ||| |   |+ 0.40% librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::finish(int) [clone .localalias] [clone .lto_priv.0]
    ||| |   | + 0.40% librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::complete_user_request(int) [clone .localalias] [clone .lto_priv.0]
    ||| |   |  + 0.40% librbd::asio::ContextWQ::queue(Context*, int)
    ||| |   |   + 0.40% pthread_cond_signal
    ||| |   + 0.20% librbd::cache::pwl::SyncPointLogOperation::appending()
    ||| |    + 0.20% librbd::cache::pwl::SyncPointLogOperation::append_sync_point()
    ||| |     + 0.20% pthread_mutex_lock
    ||| + 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::retire_entries(unsigned long)::{lambda(int)#1}>::~LambdaContext()
    |||  + 0.20% __libc_free
    |||   + 0.20% _int_free
    ||+ 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::aio_read_data_blocks(std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, Context*)::{lambda(int)#1}>::finish(int)
    |||+ 0.20% Context::complete(int)
    ||| + 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::read(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list*, int, Context*)::{lambda(int)#2}>::finish(int)
    |||  + 0.20% librbd::cache::pwl::ssd::C_ReadRequest::finish(int)
    |||   + 0.20% Context::complete(int)
    |||    + 0.20% Context::complete(int)
    |||     + 0.20% librbd::io::ReadResult::C_ImageReadRequest::finish(int)
    |||      + 0.20% Striper::StripedReadResult::add_partial_result(ceph::common::CephContext*, ceph::buffer::v15_2_0::list&&, boost::container::small_vector<std::pair<unsigned long, unsigned long>, 4ul, void, void> const&)
    |||       + 0.20% std::_Rb_tree_iterator<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > > std::_Rb_tree<unsigned long, std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> >, std::_Select1st<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<unsigned long const&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<unsigned long const, std::pair<ceph::buffer::v15_2_0::list, unsigned long> > >, std::piecewise_construct_t const&, std::tuple<unsigned long const&>&&, std::tuple<>&&) [clone .constprop.0] [clone .isra.0]
    |||        + 0.20% std::_Rb_tree<unsigned long, std::pair<unsigned long const, librbd::SnapInfo>, std::_Select1st<std::pair<unsigned long const, librbd::SnapInfo> >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, librbd::SnapInfo> > >::_M_get_insert_unique_pos(unsigned long const&) [clone .isra.0]
    ||+ 0.20% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&)::{lambda(int)#2}>::finish(int)
    || + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
    ||  + 0.20% std::condition_variable::notify_one()
    ||   + 0.20% pthread_cond_signal
    |+ 0.20% __libc_free
    | + 0.20% _int_free
    + 0.20% __libc_free
     + 0.20% _int_free

Thread 2813 (bstore_discard) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% KernelDevice::DiscardThread::entry()
   + 100.00% KernelDevice::_discard_thread()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2795 (ceph_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% ceph::timer<ceph::coarse_mono_clock>::timer_thread()
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2793 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 91.40% pthread_cond_wait
     |+ 91.20% __futex_abstimed_wait_common
     |+ 0.20% __pthread_mutex_cond_lock
     + 7.20% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 4.80% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librados::CB_AioComplete>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 2.00% librbd::image::GetMetadataRequest<librbd::ImageCtx>::handle_metadata_list(int)
     |||+ 2.00% librbd::image::GetMetadataRequest<librbd::ImageCtx>::finish(int)
     ||| + 1.20% librbd::util::detail::C_StateCallbackAdapter<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_pool_metadata, true>::complete(int)
     ||| |+ 1.00% librbd::ImageCtx::apply_metadata(std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > > const&, bool) [clone .isra.0]
     ||| ||+ 0.20% void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char const*>(char const*, char const*, std::forward_iterator_tag) [clone .isra.0]
     ||| |||+ 0.20% operator new(unsigned long)
     ||| ||| + 0.20% malloc
     ||| ||+ 0.20% ConfigValues::ConfigValues(ConfigValues const&)
     ||| |||+ 0.20% std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >* std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_M_copy<false, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node>(std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >*, std::_Rb_tree_node_base*, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node&) [clone .isra.0]
     ||| ||| + 0.20% std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >* std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_M_copy<false, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node>(std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >*, std::_Rb_tree_node_base*, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node&) [clone .isra.0]
     ||| |||  + 0.20% std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >* std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_M_copy<false, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node>(std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >*, std::_Rb_tree_node_base*, std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > >, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, std::map<int, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d>, std::less<int>, std::allocator<std::pair<int const, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> > > > > > >::_Alloc_node&) [clone .isra.0]
     ||| |||   + 0.20% operator new(unsigned long)
     ||| |||    + 0.20% malloc
     ||| |||     + 0.20% _int_malloc
     ||| ||+ 0.20% unsigned long const ceph::common::ConfigProxy::get_val<unsigned long>(std::basic_string_view<char, std::char_traits<char> >) const
     ||| |||+ 0.20% md_config_t::get_val_generic[abi:cxx11](ConfigValues const&, std::basic_string_view<char, std::char_traits<char> >) const
     ||| ||| + 0.20% md_config_t::_get_val(ConfigValues const&, std::basic_string_view<char, std::char_traits<char> >, boost::container::small_vector<std::pair<Option const*, std::variant<std::monostate, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, long, double, bool, entity_addr_t, entity_addrvec_t, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000l> >, Option::size_t, uuid_d> const*>, 4ul, void, void>*, std::ostream*) const
     ||| |||  + 0.20% ConfFile::normalize_key_name[abi:cxx11](std::basic_string_view<char, std::char_traits<char> >)
     ||| |||   + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_erase(unsigned long, unsigned long)
     ||| ||+ 0.20% librbd::io::QosImageDispatch<librbd::ImageCtx>::apply_qos_limit(unsigned long, unsigned long, unsigned long, unsigned long)
     ||| |||+ 0.20% TokenBucketThrottle::set_limit(unsigned long, unsigned long, unsigned long)
     ||| ||| + 0.20% TokenBucketThrottle::schedule_timer()
     ||| |||  + 0.20% CommonSafeTimer<std::mutex>::add_event_at(std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, Context*) [clone .localalias] [clone .lto_priv.0]
     ||| |||   + 0.20% std::_Rb_tree_iterator<std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > > > std::_Rb_tree<Context*, std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > >, std::_Select1st<std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > > >, std::less<Context*>, std::allocator<std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > > > >::_M_emplace_hint_unique<std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > >&>(std::_Rb_tree_const_iterator<std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > > >, std::pair<Context* const, std::_Rb_tree_iterator<std::pair<std::chrono::time_point<ceph::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const, Context*> > >&)
     ||| |||    + 0.20% std::_Rb_tree<ceph::md_config_obs_impl<ceph::common::ConfigProxy>*, std::pair<ceph::md_config_obs_impl<ceph::common::ConfigProxy>* const, std::unique_ptr<ceph::common::ConfigProxy::CallGate, std::default_delete<ceph::common::ConfigProxy::CallGate> > >, std::_Select1st<std::pair<ceph::md_config_obs_impl<ceph::common::ConfigProxy>* const, std::unique_ptr<ceph::common::ConfigProxy::CallGate, std::default_delete<ceph::common::ConfigProxy::CallGate> > > >, std::less<ceph::md_config_obs_impl<ceph::common::ConfigProxy>*>, std::allocator<std::pair<ceph::md_config_obs_impl<ceph::common::ConfigProxy>* const, std::unique_ptr<ceph::common::ConfigProxy::CallGate, std::default_delete<ceph::common::ConfigProxy::CallGate> > > > >::_M_get_insert_hint_unique_pos(std::_Rb_tree_const_iterator<std::pair<ceph::md_config_obs_impl<ceph::common::ConfigProxy>* const, std::unique_ptr<ceph::common::ConfigProxy::CallGate, std::default_delete<ceph::common::ConfigProxy::CallGate> > > >, ceph::md_config_obs_impl<ceph::common::ConfigProxy>* const&) [clone .isra.0]
     ||| |||     + 0.20% std::_Rb_tree_decrement(std::_Rb_tree_node_base*)
     ||| ||+ 0.20% md_config_t::set_val(ConfigValues&, ConfigTracker const&, std::basic_string_view<char, std::char_traits<char> >, char const*, std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >*)
     ||| || + 0.20% std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::find(std::basic_string_view<char, std::char_traits<char> > const&)
     ||| ||  + 0.20% __memcmp_evex_movbe
     ||| |+ 0.20% librbd::image::RefreshRequest<librbd::ImageCtx>::send_v2_get_group()
     ||| | + 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     ||| |  + 0.20% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     ||| |   + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||| |    + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||| |     + 0.20% Objecter::_calc_target(Objecter::op_target_t*, Connection*, bool) [clone .isra.0]
     ||| |      + 0.20% std::_Rb_tree<long, std::pair<long const, pg_pool_t>, std::_Select1st<std::pair<long const, pg_pool_t> >, std::less<long>, mempool::pool_allocator<(mempool::pool_index_t)23, std::pair<long const, pg_pool_t> > >::find(long const&) const [clone .isra.0]
     ||| + 0.80% librbd::util::detail::C_StateCallbackAdapter<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_metadata, true>::complete(int)
     |||  + 0.80% librbd::image::GetMetadataRequest<librbd::ImageCtx>::metadata_list()
     |||   + 0.60% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |||   |+ 0.60% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |||   | + 0.60% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |||   |  + 0.40% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |||   |  |+ 0.40% Objecter::_send_op(Objecter::Op*)
     |||   |  | + 0.20% boost::move_detail::disable_if_or<void, boost::container::dtl::is_version<boost::container::real_allocator<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >::type, 0u>, boost::move_detail::is_different<boost::container::real_allocator<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >::type, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >, boost::move_detail::bool_<false>, boost::move_detail::bool_<false> >::type boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void>::priv_copy_assign<boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void> >(boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void> const&)
     |||   |  | |+ 0.20% ceph::buffer::v15_2_0::ptr_node::cloner::operator()(ceph::buffer::v15_2_0::ptr_node const&)
     |||   |  | | + 0.20% operator new(unsigned long)
     |||   |  | |  + 0.20% malloc
     |||   |  | |   + 0.20% _int_malloc
     |||   |  | + 0.20% AsyncConnection::send_message(Message*)
     |||   |  |  + 0.20% ProtocolV2::send_message(Message*)
     |||   |  |   + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     |||   |  |    + 0.20% EventCenter::wakeup()
     |||   |  |     + 0.20% __write
     |||   |  + 0.20% Objecter::_op_submit_with_budget(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*, int*)
     |||   |   + 0.20% Throttle::get_or_fail(long)
     |||   |    + 0.20% ceph::common::PerfCounters::set(int, unsigned long)
     |||   + 0.20% ceph::buffer::v15_2_0::list::append(char const*, unsigned int)
     |||    + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     |||     + 0.20% posix_memalign
     |||      + 0.20% malloc
     |||       + 0.20% _int_malloc
     |||        + 0.20% malloc_consolidate
     ||+ 1.00% void librbd::util::detail::rados_state_callback<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent, true>(void*, void*)
     |||+ 1.00% librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent(int*)
     ||| + 0.60% librbd::image::GetMetadataRequest<librbd::ImageCtx>::metadata_list()
     ||| |+ 0.40% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     ||| ||+ 0.40% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     ||| || + 0.40% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||| ||  + 0.40% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||| ||   + 0.20% std::unique_lock<std::shared_mutex>::lock()
     ||| ||   |+ 0.20% __pthread_rwlock_wrlock
     ||| ||   | + 0.20% __futex_abstimed_wait_common
     ||| ||   + 0.20% Objecter::_calc_target(Objecter::op_target_t*, Connection*, bool) [clone .isra.0]
     ||| ||    + 0.20% OSDMap::map_to_pg(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, pg_t*) const
     ||| ||     + 0.20% pg_pool_t::hash_key(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const
     ||| ||      + 0.20% ceph_str_hash_rjenkins(char const*, unsigned int)
     ||| |+ 0.20% ceph::buffer::v15_2_0::list::append(char const*, unsigned int)
     ||| | + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     ||| |  + 0.20% posix_memalign
     ||| |   + 0.20% malloc
     ||| |    + 0.20% _int_malloc
     ||| |     + 0.20% malloc_consolidate
     ||| + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::iterator_impl(ceph::buffer::v15_2_0::list const*, unsigned int) [clone .localalias] [clone .lto_priv.0]
     ||+ 0.80% librbd::operation::Request<librbd::ImageCtx>::finish_and_destroy(int)
     |||+ 0.80% librbd::ImageWatcher<librbd::ImageCtx>::send_notify(librbd::watch_notify::Payload*, Context*)
     ||| + 0.60% librbd::watcher::Notifier::notify(ceph::buffer::v15_2_0::list&, librbd::watcher::NotifyResponse*, Context*)
     ||| |+ 0.60% librados::v14_2_0::IoCtx::aio_notify(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, ceph::buffer::v15_2_0::list&, unsigned long, ceph::buffer::v15_2_0::list*)
     ||| | + 0.40% librados::IoCtxImpl::aio_notify(object_t const&, librados::AioCompletionImpl*, ceph::buffer::v15_2_0::list&, unsigned long, ceph::buffer::v15_2_0::list*, char**, unsigned long*)
     ||| |  + 0.20% Objecter::linger_notify(Objecter::LingerOp*, ObjectOperation&, snapid_t, ceph::buffer::v15_2_0::list&, std::unique_ptr<ceph::async::Completion<void (boost::system::error_code, ceph::buffer::v15_2_0::list), void>, std::default_delete<ceph::async::Completion<void (boost::system::error_code, ceph::buffer::v15_2_0::list), void> > >&&, unsigned long*) [clone .isra.0]
     ||| |  |+ 0.20% Objecter::_linger_submit(Objecter::LingerOp*, ceph::shunique_lock<std::shared_mutex>&)
     ||| |  | + 0.20% Objecter::_send_linger(Objecter::LingerOp*, ceph::shunique_lock<std::shared_mutex>&)
     ||| |  |  + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||| |  |   + 0.20% Objecter::_send_op_account(Objecter::Op*)
     ||| |  |    + 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     ||| |  + 0.20% ceph::buffer::v15_2_0::list::buffers_t::clear_and_dispose()
     ||| |   + 0.20% __libc_free
     ||| + 0.20% librbd::watch_notify::NotifyMessage::encode(ceph::buffer::v15_2_0::list&) const
     |||  + 0.20% ceph::buffer::v15_2_0::list::append_hole(unsigned int)
     |||   + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     |||    + 0.20% posix_memalign
     |||     + 0.20% malloc
     |||      + 0.20% _int_malloc
     |||       + 0.20% malloc_consolidate
     ||+ 0.80% void librbd::util::detail::rados_state_callback<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_mutable_metadata, true>(void*, void*)
     |||+ 0.80% librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_mutable_metadata(int*)
     ||| + 0.40% librbd::image::RefreshRequest<librbd::ImageCtx>::send_v2_get_parent()
     ||| |+ 0.20% librbd::cls_client::parent_overlap_get_start(librados::v14_2_0::ObjectReadOperation*, snapid_t)
     ||| ||+ 0.20% librados::v14_2_0::ObjectOperation::exec(char const*, char const*, ceph::buffer::v15_2_0::list&)
     ||| || + 0.20% ceph::buffer::v15_2_0::list::append(char const*, unsigned int)
     ||| ||  + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
     ||| ||   + 0.20% posix_memalign
     ||| ||    + 0.20% malloc
     ||| ||     + 0.20% _int_malloc
     ||| |+ 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     ||| | + 0.20% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     ||| |  + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||| |   + 0.20% Objecter::_op_submit_with_budget(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*, int*)
     ||| |    + 0.20% Throttle::get_or_fail(long)
     ||| + 0.20% librbd::cls_client::get_size_finish(ceph::buffer::v15_2_0::list::iterator_impl<true>*, unsigned long*, unsigned char*)
     ||| |+ 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, char*) [clone .localalias] [clone .lto_priv.0]
     ||| + 0.20% rados::cls::lock::get_lock_info_finish(ceph::buffer::v15_2_0::list::iterator_impl<true>*, std::map<rados::cls::lock::locker_id_t, rados::cls::lock::locker_info_t, std::less<rados::cls::lock::locker_id_t>, std::allocator<std::pair<rados::cls::lock::locker_id_t const, rados::cls::lock::locker_info_t> > >*, ClsLockType*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*)
     |||  + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) [clone .localalias] [clone .lto_priv.0]
     |||   + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long)
     |||    + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)
     |||     + 0.20% _init
     ||+ 0.20% void librbd::util::detail::rados_callback<librbd::watcher::Notifier::C_AioNotify>(void*, void*)
     || + 0.20% librbd::watcher::Notifier::C_AioNotify::finish(int)
     ||  + 0.20% librbd::asio::ContextWQ::queue(Context*, int)
     ||   + 0.20% pthread_cond_signal
     |+ 2.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     |||+ 1.00% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||| + 0.80% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     ||| |+ 0.80% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*) [clone .localalias]
     ||| | + 0.80% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
     ||| |  + 0.40% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::alloc_and_dispatch_io_req(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| |  |+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::alloc_resources(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| |  | + 0.40% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::check_allocation(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, unsigned long, unsigned long, unsigned long, unsigned int, unsigned int, unsigned int) [clone .localalias] [clone .lto_priv.0]
     ||| |  |  + 0.40% librbd::cache::pwl::ImageCacheState<librbd::ImageCtx>::write_image_cache_state(std::unique_lock<std::mutex>&, Context*) [clone .localalias] [clone .lto_priv.0]
     ||| |  |   + 0.40% librbd::operation::MetadataSetRequest<librbd::ImageCtx>::send_op()
     ||| |  |    + 0.40% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectWriteOperation*)
     ||| |  |     + 0.40% librados::IoCtxImpl::aio_operate(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, SnapContext const&, int, blkin_trace_info const*) [clone .isra.0]
     ||| |  |      + 0.20% Objecter::prepare_mutate_op(object_t const&, object_locator_t const&, ObjectOperation&, SnapContext const&, std::chrono::time_point<ceph::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, int, Context*, unsigned long*, osd_reqid_t, ZTracer::Trace*) [clone .constprop.0] [clone .isra.0]
     ||| |  |      |+ 0.20% void boost::container::vector<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::small_vector_allocator<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::new_allocator<void>, void>, void>::priv_swap<boost::container::vector<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::small_vector_allocator<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::new_allocator<void>, void>, void> >(boost::container::vector<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::small_vector_allocator<fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >, boost::container::new_allocator<void>, void>, void>&, boost::move_detail::integral_constant<bool, false>) [clone .isra.0]
     ||| |  |      | + 0.20% fu2::abi_310::detail::type_erasure::tables::vtable<fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >::empty_cmd(fu2::abi_310::detail::type_erasure::tables::vtable<fu2::abi_310::detail::property<true, false, void (boost::system::error_code, int, ceph::buffer::v15_2_0::list const&) &&> >*, fu2::abi_310::detail::type_erasure::tables::opcode, fu2::abi_310::detail::type_erasure::data_accessor*, unsigned long, fu2::abi_310::detail::type_erasure::data_accessor*, unsigned long)
     ||| |  |      + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||| |  |       + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||| |  |        + 0.20% Objecter::_send_op(Objecter::Op*)
     ||| |  |         + 0.20% AsyncConnection::send_message(Message*)
     ||| |  |          + 0.20% ProtocolV2::send_message(Message*)
     ||| |  |           + 0.20% EventCenter::dispatch_event_external(EventCallback*)
     ||| |  |            + 0.20% EventCenter::wakeup()
     ||| |  |             + 0.20% __write
     ||| |  + 0.40% librbd::cache::pwl::C_WriteRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::dispatch() [clone .localalias] [clone .lto_priv.0]
     ||| |   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::schedule_append(std::vector<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| |    + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::schedule_append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
     ||| |     + 0.20% C_GatherBase<Context, Context>::C_GatherSub::complete(int)
     ||| |      + 0.20% C_GatherBase<Context, Context>::C_GatherSub::finish(int)
     ||| |       + 0.20% C_GatherBase<Context, Context>::sub_finish(Context*, int)
     ||| |        + 0.20% pthread_mutex_lock
     ||| + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::read(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, librbd::io::ReadResult&&, std::shared_ptr<neorados::IOContext>, int, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |||  + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::read(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list*, int, Context*) [clone .localalias] [clone .lto_priv.0]
     |||   + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::collect_read_extents(unsigned long, librbd::cache::pwl::LogMapEntry<librbd::cache::pwl::GenericWriteLogEntry>, std::vector<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericWriteLogEntry> > >&, std::vector<ceph::buffer::v15_2_0::list*, std::allocator<ceph::buffer::v15_2_0::list*> >&, unsigned long, std::pair<unsigned long, unsigned long>, librbd::cache::pwl::C_ReadRequest*) [clone .localalias] [clone .lto_priv.0]
     |||    + 0.20% ceph::logging::Log::submit_entry(ceph::logging::Entry&&)
     |||     + 0.20% std::condition_variable::notify_all()
     |||      + 0.20% pthread_cond_broadcast
     ||+ 1.00% Context::complete(int)
     |||+ 0.60% LambdaContext<librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool)::{lambda(int)#1}::operator()(int) const::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}::operator()(librbd::cache::pwl::GuardedRequestFunctionContext&) const::{lambda(int)#1}>::finish(int)
     ||||+ 0.60% librbd::cache::pwl::ssd::WriteLogEntry::writeback_bl(librbd::cache::ImageWritebackInterface&, Context*, ceph::buffer::v15_2_0::list&&)
     |||| + 0.60% librbd::cache::ImageWriteback<librbd::ImageCtx>::aio_write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*)
     ||||  + 0.60% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     ||||   + 0.60% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||||    + 0.60% librbd::io::ImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
     ||||     + 0.60% librbd::io::ImageRequest<librbd::ImageCtx>::aio_write(librbd::ImageCtx*, librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&)
     ||||      + 0.60% librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>::send_request()
     ||||       + 0.60% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ObjectDispatcherInterface>::send(librbd::io::ObjectDispatchSpec*)
     ||||        + 0.40% librbd::io::ObjectDispatch<librbd::ImageCtx>::write(unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, int, std::optional<unsigned long>, ZTracer::Trace const&, int*, unsigned long*, librbd::io::DispatchResult*, Context**, Context*)
     ||||        |+ 0.40% librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()
     ||||        | + 0.40% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     ||||        |  + 0.20% Throttle::get_or_fail(long)
     ||||        |  |+ 0.20% ceph::common::PerfCounters::inc(int, unsigned long)
     ||||        |  + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     ||||        + 0.20% AsyncOpTracker::finish_op()
     ||||         + 0.20% pthread_mutex_lock
     |||+ 0.40% librbd::io::AioCompletion::complete_request(long)
     ||| + 0.40% librbd::io::AioCompletion::complete()
     |||  + 0.20% boost::asio::detail::strand_service::do_dispatch(boost::asio::detail::strand_service::strand_impl*&, boost::asio::detail::scheduler_operation*)
     |||  |+ 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::io::AioCompletion::complete_external_callback()::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |||  | + 0.20% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
     |||  |  + 0.20% __libc_free
     |||  |   + 0.20% _int_free
     |||  + 0.20% void librbd::io::AioCompletion::callback_adapter<Context, &Context::complete>(void*, void*)
     |||   + 0.20% Context::complete(int)
     |||    + 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::construct_flush_entry(std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, bool)::{lambda(int)#1}>::finish(int)
     |||     + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
     |||      + 0.20% std::condition_variable::notify_one()
     |||       + 0.20% pthread_cond_signal
     ||+ 0.20% librbd::util::detail::C_StateCallbackAdapter<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_apply, true>::complete(int)
     || + 0.20% librbd::image::RefreshRequest<librbd::ImageCtx>::~RefreshRequest() [clone .lto_priv.0]
     ||  + 0.20% std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >*) [clone .isra.0]
     ||   + 0.20% std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> >*) [clone .isra.0]
     |+ 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<CB_DoWatchNotify>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     | + 0.20% WatchInfo::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |  + 0.20% librbd::Watcher::WatchCtx::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |   + 0.20% librbd::ImageWatcher<librbd::ImageCtx>::handle_notify(unsigned long, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&)
     |    + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, char*) [clone .localalias] [clone .lto_priv.0]
     |     + 0.20% ceph::buffer::v15_2_0::ptr::copy_out(unsigned int, unsigned int, char*) const
     + 1.00% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 1.00% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 1.00% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |  + 0.80% librbd::io::RefreshImageDispatch<librbd::ImageCtx>::needs_refresh(librbd::io::DispatchResult*, Context*)
     |  |+ 0.80% librbd::ImageState<librbd::ImageCtx>::refresh(Context*)
     |  | + 0.80% librbd::image::RefreshRequest<librbd::ImageCtx>::send_v2_get_mutable_metadata()
     |  |  + 0.40% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |  |  |+ 0.40% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |  |  | + 0.40% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |  |  |  + 0.20% __pthread_rwlock_rdlock
     |  |  |  |+ 0.20% __futex_abstimed_wait_common
     |  |  |  + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |  |  |   + 0.20% Objecter::_send_op(Objecter::Op*)
     |  |  |    + 0.20% AsyncConnection::send_message(Message*)
     |  |  |     + 0.20% ProtocolV2::send_message(Message*)
     |  |  |      + 0.20% ceph::common::RefCountedObject::_get() const
     |  |  + 0.20% rados::cls::lock::get_lock_info_start(librados::v14_2_0::ObjectReadOperation*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
     |  |  |+ 0.20% ceph::buffer::v15_2_0::list::append(ceph::buffer::v15_2_0::list const&)
     |  |  | + 0.20% operator new(unsigned long)
     |  |  |  + 0.20% malloc
     |  |  + 0.20% librbd::cls_client::get_flags_start(librados::v14_2_0::ObjectReadOperation*, snapid_t)
     |  |   + 0.20% librados::v14_2_0::ObjectOperation::exec(char const*, char const*, ceph::buffer::v15_2_0::list&)
     |  |    + 0.20% ObjectOperation::add_op(int)
     |  |     + 0.20% boost::container::vec_iterator<ceph::buffer::v15_2_0::list**, false> boost::container::vector<ceph::buffer::v15_2_0::list*, boost::container::small_vector_allocator<ceph::buffer::v15_2_0::list*, boost::container::new_allocator<void>, void>, void>::priv_insert_forward_range_no_capacity<boost::container::dtl::insert_emplace_proxy<boost::container::small_vector_allocator<ceph::buffer::v15_2_0::list*, boost::container::new_allocator<void>, void>, ceph::buffer::v15_2_0::list**, ceph::buffer::v15_2_0::list*> >(ceph::buffer::v15_2_0::list**, unsigned long, boost::container::dtl::insert_emplace_proxy<boost::container::small_vector_allocator<ceph::buffer::v15_2_0::list*, boost::container::new_allocator<void>, void>, ceph::buffer::v15_2_0::list**, ceph::buffer::v15_2_0::list*>, boost::move_detail::integral_constant<unsigned int, 1u>) [clone .constprop.0]
     |  + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*) [clone .localalias]
     + 0.20% boost::asio::detail::executor_op<ceph::async::ForwardingHandler<ceph::async::CompletionHandler<librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, std::tuple<boost::system::error_code> > >, std::allocator<ceph::async::detail::CompletionImpl<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>, librbd::asio::util::get_callback_adapter<librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}>(librbd::io::AbstractObjectWriteRequest<librbd::ImageCtx>::write_object()::{lambda(int)#1}&&)::{lambda(boost::system::error_code, (auto:1&&)...)#1}, void, boost::system::error_code> >, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
      + 0.20% librbd::io::ObjectRequest<librbd::ImageCtx>::finish(int)
       + 0.20% librbd::io::ObjectDispatchSpec::C_Dispatcher::finish(int)
        + 0.20% Context::complete(int)
         + 0.20% LambdaContext<librbd::io::SimpleSchedulerObjectDispatch<librbd::ImageCtx>::register_in_flight_request(unsigned long, utime_t const&, Context**)::{lambda(int)#1}>::finish(int)
          + 0.20% Context::complete(int)
           + 0.20% librbd::io::AioCompletion::complete_request(long)
            + 0.20% librbd::io::AioCompletion::complete()
             + 0.20% void librbd::io::AioCompletion::callback_adapter<Context, &Context::complete>(void*, void*)
              + 0.20% Context::complete(int)
               + 0.20% librbd::cache::ImageWriteback<librbd::ImageCtx>::aio_flush(librbd::io::FlushSource, Context*)
                + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
                 + 0.20% librbd::io::ImageDispatch<librbd::ImageCtx>::get_dispatch_layer() const

Thread 2794 (io_context_pool) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::thread::_State_impl<std::thread::_Invoker<std::tuple<make_named_thread<ceph::async::io_context_pool::start(short)::{lambda()#1}>(std::basic_string_view<char, std::char_traits<char> >, ceph::async::io_context_pool::start(short)::{lambda()#1}&&)::{lambda(auto:1&&, (auto:2&&)...)#1}, ceph::async::io_context_pool::start(short)::{lambda()#1}> > >::_M_run()
    + 100.00% boost::asio::detail::scheduler::run(boost::system::error_code&) [clone .constprop.0] [clone .isra.0]
     + 98.40% pthread_cond_wait
     |+ 97.80% __futex_abstimed_wait_common
     |+ 0.20% __pthread_mutex_cond_lock
     |+ 0.20% __condvar_dec_grefs
     + 0.80% boost::asio::detail::strand_service::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.40% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librbd::asio::ContextWQ::queue(Context*, int)::{lambda()#1}>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     ||+ 0.20% Context::complete(int)
     |||+ 0.20% librbd::io::AioCompletion::complete_request(long)
     ||| + 0.20% librbd::io::AioCompletion::complete()
     |||  + 0.20% void librbd::io::AioCompletion::callback_adapter<Context, &Context::complete>(void*, void*)
     |||   + 0.20% Context::complete(int)
     |||    + 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::construct_flush_entry(std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, bool)::{lambda(int)#1}>::finish(int)
     |||     + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
     |||      + 0.20% std::condition_variable::notify_one()
     |||       + 0.20% pthread_cond_signal
     ||+ 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     || + 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     ||  + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     ||   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*) [clone .localalias]
     ||    + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
     ||     + 0.20% boost::detail::function::void_function_obj_invoker1<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*)::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}, void, librbd::cache::pwl::GuardedRequestFunctionContext&>::invoke(boost::detail::function::function_buffer&, librbd::cache::pwl::GuardedRequestFunctionContext&)
     ||      + 0.20% librbd::cache::pwl::C_WriteRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::blockguard_acquired(librbd::cache::pwl::GuardedRequestFunctionContext&) [clone .localalias] [clone .lto_priv.0]
     |+ 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librados::(anonymous namespace)::CB_aio_linger_cancel>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long) [clone .lto_priv.0]
     ||+ 0.20% Objecter::linger_cancel(Objecter::LingerOp*)
     || + 0.20% Objecter::_linger_cancel(Objecter::LingerOp*)
     ||  + 0.20% std::_Rb_tree<Objecter::LingerOp*, Objecter::LingerOp*, std::_Identity<Objecter::LingerOp*>, std::less<Objecter::LingerOp*>, std::allocator<Objecter::LingerOp*> >::erase(Objecter::LingerOp* const&) [clone .isra.0]
     |+ 0.20% boost::asio::detail::completion_handler<boost::asio::detail::binder0<librados::CB_AioComplete>, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     | + 0.20% void librbd::util::detail::rados_state_callback<librbd::image::RefreshRequest<librbd::ImageCtx>, &librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent, true>(void*, void*)
     |  + 0.20% librbd::image::RefreshRequest<librbd::ImageCtx>::handle_v2_get_parent(int*)
     |   + 0.20% librbd::image::GetMetadataRequest<librbd::ImageCtx>::metadata_list()
     |    + 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |     + 0.20% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |      + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |       + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |        + 0.20% Objecter::_calc_target(Objecter::op_target_t*, Connection*, bool) [clone .isra.0]
     |         + 0.20% std::_Rb_tree<long, std::pair<long const, pg_pool_t>, std::_Select1st<std::pair<long const, pg_pool_t> >, std::less<long>, mempool::pool_allocator<(mempool::pool_index_t)23, std::pair<long const, pg_pool_t> > >::find(long const&) const [clone .isra.0]
     + 0.60% boost::asio::detail::executor_op<boost::asio::detail::binder0<librbd::AsioEngine::post(Context*, int)::{lambda()#1}>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)
     |+ 0.60% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
     | + 0.60% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
     |  + 0.40% librbd::io::RefreshImageDispatch<librbd::ImageCtx>::needs_refresh(librbd::io::DispatchResult*, Context*)
     |  |+ 0.40% librbd::ImageState<librbd::ImageCtx>::refresh(Context*)
     |  | + 0.40% librbd::image::RefreshRequest<librbd::ImageCtx>::send_v2_get_mutable_metadata()
     |  |  + 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectReadOperation*, ceph::buffer::v15_2_0::list*)
     |  |  |+ 0.20% librados::IoCtxImpl::aio_operate_read(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, int, ceph::buffer::v15_2_0::list*, blkin_trace_info const*) [clone .isra.0]
     |  |  | + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     |  |  |  + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     |  |  |   + 0.20% Objecter::_get_session(int, Objecter::OSDSession**, ceph::shunique_lock<std::shared_mutex>&) [clone .isra.0]
     |  |  |    + 0.20% ceph::common::RefCountedObject::_get() const
     |  |  + 0.20% librbd::cls_client::get_size_start(librados::v14_2_0::ObjectReadOperation*, snapid_t)
     |  |   + 0.20% librados::v14_2_0::ObjectOperation::exec(char const*, char const*, ceph::buffer::v15_2_0::list&)
     |  |    + 0.20% __strlen_evex
     |  + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
     |   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::write(std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, int, Context*) [clone .localalias]
     |    + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
     |     + 0.20% librbd::cache::pwl::C_WriteRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::dispatch() [clone .localalias] [clone .lto_priv.0]
     |      + 0.20% librbd::cache::pwl::C_WriteRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::setup_log_operations(librbd::cache::pwl::DeferredContexts&) [clone .localalias] [clone .lto_priv.0]
     |       + 0.20% librbd::cache::pwl::WriteLogOperationSet::WriteLogOperationSet(utime_t, ceph::common::PerfCounters*, std::shared_ptr<librbd::cache::pwl::SyncPoint>, bool, ceph::common::CephContext*, Context*)
     |        + 0.20% operator new(unsigned long)
     |         + 0.20% malloc
     + 0.20% pthread_mutex_lock
      + 0.20% __GI___lll_lock_wait

Thread 2785 (log) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::logging::Log::entry()
   + 99.40% std::condition_variable::wait(std::unique_lock<std::mutex>&)
   |+ 99.40% pthread_cond_wait
   | + 99.40% __futex_abstimed_wait_common
   + 0.60% __GI___pthread_mutex_unlock_usercnt
    + 0.60% __GI___lll_lock_wake

Thread 2796 (ms_dispatch) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::DispatchThread::entry()
   + 100.00% DispatchQueue::entry()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2797 (ms_local) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% DispatchQueue::LocalDeliveryThread::entry()
   + 100.00% DispatchQueue::run_local_delivery()
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2786 (msgr-worker-0) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 98.20% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 98.20% epoll_wait
     + 1.20% ProtocolV2::write_event()
     |+ 0.80% ProtocolV2::write_message(Message*, bool)
     ||+ 0.80% AsyncConnection::_try_send(bool)
     || + 0.80% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
     ||  + 0.80% sendmsg
     |+ 0.20% ProtocolV2::_get_next_outgoing()
     ||+ 0.20% std::_Rb_tree<int, std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > >, std::_Select1st<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > >, std::less<int>, std::allocator<std::pair<int const, std::__cxx11::list<ProtocolV2::out_queue_entry_t, std::allocator<ProtocolV2::out_queue_entry_t> > > > >::erase(int const&) [clone .isra.0]
     |+ 0.20% Message::encode(unsigned long, int, bool)
     | + 0.20% _mosdop::MOSDOp<boost::container::small_vector<OSDOp, 2ul, void, void> >::encode_payload(unsigned long)
     |  + 0.20% ceph::buffer::v15_2_0::ptr::append(char const*, unsigned int)
     + 0.60% AsyncConnection::process()
      + 0.60% ProtocolV2::run_continuation(Ct<ProtocolV2>&)
       + 0.60% ProtocolV2::handle_message()
        + 0.20% DispatchQueue::fast_dispatch(Message*)
        |+ 0.20% DispatchQueue::fast_dispatch(boost::intrusive_ptr<Message> const&)
        | + 0.20% Dispatcher::ms_fast_dispatch2(boost::intrusive_ptr<Message> const&)
        |  + 0.20% Objecter::ms_dispatch(Message*)
        |   + 0.20% Objecter::handle_osd_op_reply(MOSDOpReply*)
        |    + 0.20% Objecter::Op::~Op()
        |     + 0.20% boost::container::vector<OSDOp, boost::container::small_vector_allocator<OSDOp, boost::container::new_allocator<void>, void>, void>::~vector() [clone .lto_priv.0]
        |      + 0.20% __libc_free
        |       + 0.20% _int_free
        + 0.20% decode_message(ceph::common::CephContext*, int, ceph_msg_header&, ceph_msg_footer&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&, boost::intrusive_ptr<Connection>)
         + 0.20% MOSDOpReply::decode_payload()
          + 0.20% OSDOp::split_osd_op_vector_out_data(std::vector<OSDOp, std::allocator<OSDOp> >&, ceph::buffer::v15_2_0::list&)
           + 0.20% ceph::buffer::v15_2_0::list::iterator_impl<true>::copy(unsigned int, ceph::buffer::v15_2_0::list&) [clone .localalias] [clone .lto_priv.0]
            + 0.20% ceph::buffer::v15_2_0::list::append(ceph::buffer::v15_2_0::ptr const&, unsigned int, unsigned int)
             + 0.20% ceph::buffer::v15_2_0::ptr::ptr(ceph::buffer::v15_2_0::ptr const&, unsigned int, unsigned int)

Thread 2787 (msgr-worker-1) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 99.80% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
     |+ 99.80% epoll_wait
     + 0.20% ProtocolV2::write_event()
      + 0.20% ProtocolV2::write_message(Message*, bool)
       + 0.20% AsyncConnection::_try_send(bool)
        + 0.20% PosixConnectedSocketImpl::send(ceph::buffer::v15_2_0::list&, bool)
         + 0.20% sendmsg

Thread 2788 (msgr-worker-2) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% std::_V2::error_category::_M_message[abi:cxx11](int) const
   + 100.00% std::_Function_handler<void (), NetworkStack::add_thread(Worker*)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
    + 100.00% EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)
     + 100.00% EpollDriver::event_wait(std::vector<FiredFileEvent, std::allocator<FiredFileEvent> >&, timeval*)
      + 100.00% epoll_wait

Thread 2762 (qemu-system-x86) - 500 samples
+ 99.00% _start
|+ 99.00% __libc_start_main
| + 99.00% __libc_start_call_main
|  + 99.00% qemu_default_main
|   + 99.00% qemu_main_loop
|    + 98.80% main_loop_wait
|     + 98.00% qemu_poll_ns
|     |+ 98.00% ppoll
|     + 0.20% g_main_context_query
|     + 0.20% g_main_context_check
|     |+ 0.20% aio_ctx_check
|     | + 0.20% aio_pending
|     + 0.20% timerlistgroup_deadline_ns
|     |+ 0.20% timerlist_deadline_ns
|     | + 0.20% qemu_mutex_lock_impl
|     |  + 0.20% pthread_mutex_lock
|     + 0.20% g_main_context_dispatch
|      + 0.20% aio_ctx_dispatch
|       + 0.20% aio_dispatch
|        + 0.20% aio_dispatch_handler
|         + 0.20% virtio_queue_notify_vq.part.0
|          + 0.20% virtio_blk_handle_vq
|           + 0.20% virtqueue_split_pop
|            + 0.20% virtqueue_alloc_element
|             + 0.20% g_malloc
|              + 0.20% malloc
|               + 0.20% _int_malloc
+ 1.00% __push___start_context
 + 1.00% coroutine_trampoline
  + 0.60% blk_aio_write_entry
  |+ 0.40% blk_co_do_pwritev_part
  ||+ 0.40% bdrv_co_pwritev_part
  || + 0.40% bdrv_aligned_pwritev
  ||  + 0.40% bdrv_driver_pwritev
  ||   + 0.40% raw_co_pwritev
  ||    + 0.40% bdrv_co_pwritev_part
  ||     + 0.40% bdrv_aligned_pwritev
  ||      + 0.40% bdrv_driver_pwritev
  ||       + 0.40% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
  ||        + 0.40% rbd_aio_writev
  ||         + 0.20% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
  ||         |+ 0.20% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
  ||         | + 0.20% librbd::io::QueueImageDispatch<librbd::ImageCtx>::write(librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
  ||         + 0.20% librbd::api::Io<librbd::ImageCtx>::aio_write(librbd::ImageCtx&, librbd::io::AioCompletion*, unsigned long, unsigned long, ceph::buffer::v15_2_0::list&&, int, bool)
  ||          + 0.20% librbd::io::ImageDispatchSpec* librbd::io::ImageDispatchSpec::create_write<librbd::ImageCtx>(librbd::ImageCtx&, librbd::io::ImageDispatchLayer, librbd::io::AioCompletion*, std::vector<std::pair<unsigned long, unsigned long>, std::allocator<std::pair<unsigned long, unsigned long> > >&&, ceph::buffer::v15_2_0::list&&, std::shared_ptr<neorados::IOContext>, int, ZTracer::Trace const&)
  ||           + 0.20% operator new(unsigned long)
  ||            + 0.20% malloc
  |+ 0.20% virtio_blk_rw_complete
  | + 0.20% virtio_notify_irqfd
  + 0.40% blk_aio_flush_entry
   + 0.40% bdrv_co_flush
    + 0.40% bdrv_co_flush
     + 0.40% qemu_stamp_68c239b247a1538c1203e157c9063a542caf3f8b
      + 0.40% rbd_aio_flush
       + 0.40% librbd::api::Io<librbd::ImageCtx>::aio_flush(librbd::ImageCtx&, librbd::io::AioCompletion*, bool)
        + 0.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
         + 0.40% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
          + 0.40% librbd::io::QueueImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
           + 0.40% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
            + 0.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
             + 0.40% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
              + 0.40% librbd::io::QosImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*)
               + 0.40% librbd::io::FlushTracker<librbd::ImageCtx>::flush(Context*)
                + 0.40% librbd::io::Dispatcher<librbd::ImageCtx, librbd::io::ImageDispatcherInterface>::send(librbd::io::ImageDispatchSpec*)
                 + 0.40% librbd::io::ImageDispatcher<librbd::ImageCtx>::send_dispatch(librbd::io::ImageDispatchInterface*, librbd::io::ImageDispatchSpec*)
                  + 0.20% librbd::io::RefreshImageDispatch<librbd::ImageCtx>::needs_refresh(librbd::io::DispatchResult*, Context*)
                  |+ 0.20% librbd::ImageState<librbd::ImageCtx>::refresh(Context*)
                  | + 0.20% librbd::ImageState<librbd::ImageCtx>::execute_next_action_unlock()
                  |  + 0.20% librbd::image::RefreshRequest<librbd::ImageCtx>::RefreshRequest(librbd::ImageCtx&, bool, bool, Context*) [clone .lto_priv.0]
                  + 0.20% librbd::cache::WriteLogImageDispatch<librbd::ImageCtx>::flush(librbd::io::AioCompletion*, librbd::io::FlushSource, ZTracer::Trace const&, unsigned long, std::atomic<unsigned int>*, librbd::io::DispatchResult*, Context**, Context*) [clone .localalias]
                   + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::detain_guarded_request(librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*, librbd::cache::pwl::GuardedRequestFunctionContext*, bool) [clone .localalias] [clone .lto_priv.0]
                    + 0.20% boost::detail::function::void_function_obj_invoker1<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::flush(librbd::io::FlushSource, Context*)::{lambda(librbd::cache::pwl::GuardedRequestFunctionContext&)#1}, void, librbd::cache::pwl::GuardedRequestFunctionContext&>::invoke(boost::detail::function::function_buffer&, librbd::cache::pwl::GuardedRequestFunctionContext&)
                     + 0.20% librbd::cache::pwl::DeferredContexts::~DeferredContexts()
                      + 0.20% Context::complete(int)
                       + 0.20% librbd::cache::pwl::SyncPoint::persist_gather_activate()
                        + 0.20% Context::complete(int)
                         + 0.20% librbd::cache::pwl::C_FlushRequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >::dispatch() [clone .localalias] [clone .lto_priv.0]
                          + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::schedule_append(std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
                           + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::schedule_append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, librbd::cache::pwl::C_BlockIORequest<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx> >*) [clone .localalias] [clone .lto_priv.0]
                            + 0.20% librbd::cache::pwl::SyncPointLogOperation::appending()
                             + 0.20% librbd::cache::pwl::SyncPointLogOperation::append_sync_point()
                              + 0.20% pthread_mutex_lock
                               + 0.20% __GI___lll_lock_wait

Thread 2784 (qemu-system-x86) - 500 samples
+ 100.00% __clone3
 + 100.00% start_thread
  + 100.00% qemu_thread_start
   + 100.00% call_rcu_thread
    + 100.00% qemu_event_wait
     + 100.00% syscall

Thread 2798 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2799 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2801 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
     + 100.00% pthread_cond_wait
      + 100.00% __futex_abstimed_wait_common

Thread 2800 (safe_timer) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% CommonSafeTimerThread<std::mutex>::entry()
   + 100.00% CommonSafeTimer<std::mutex>::timer_thread() [clone .localalias]
    + 100.00% pthread_cond_clockwait
     + 100.00% __futex_abstimed_wait_common

Thread 2792 (service) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ceph::common::CephContextServiceThread::entry()
   + 100.00% pthread_cond_clockwait
    + 100.00% __futex_abstimed_wait_common

Thread 2802 (taskfin_librbd) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% Finisher::finisher_thread_entry()
   + 100.00% std::condition_variable::wait(std::unique_lock<std::mutex>&)
    + 100.00% pthread_cond_wait
     + 100.00% __futex_abstimed_wait_common

Thread 2815 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 98.20% pthread_cond_clockwait
    |+ 98.00% __futex_abstimed_wait_common
    |+ 0.20% __pthread_mutex_cond_lock
    | + 0.20% __GI___lll_lock_wait
    + 1.80% Context::complete(int)
     + 0.80% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
     |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
     ||+ 0.40% KernelDevice::aio_submit(IOContext*)
     |||+ 0.40% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
     ||| + 0.40% io_submit
     |||  + 0.40% syscall
     ||+ 0.20% ceph::buffer::v15_2_0::list::obtain_contiguous_space(unsigned int)
     || + 0.20% ceph::buffer::v15_2_0::create(unsigned int)
     ||  + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
     ||   + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
     ||    + 0.20% ceph::buffer::v15_2_0::raw_combined::create(unsigned int, unsigned int, int)
     ||     + 0.20% posix_memalign
     ||      + 0.20% _mid_memalign.isra.0
     |+ 0.20% std::recursive_mutex::lock()
     | + 0.20% pthread_mutex_lock
     |  + 0.20% __GI___lll_lock_wait
     + 0.60% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
     |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
     | + 0.40% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
     | |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::construct_flush_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::DeferredContexts&, bool) [clone .localalias] [clone .lto_priv.0]
     | ||+ 0.20% KernelDevice::aio_submit(IOContext*)
     | || + 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
     | ||  + 0.20% io_submit
     | ||   + 0.20% syscall
     | |+ 0.20% librbd::cache::pwl::ImageCacheState<librbd::ImageCtx>::write_image_cache_state(std::unique_lock<std::mutex>&, Context*) [clone .localalias] [clone .lto_priv.0]
     | | + 0.20% librbd::operation::MetadataSetRequest<librbd::ImageCtx>::send_op()
     | |  + 0.20% librados::v14_2_0::IoCtx::aio_operate(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, librados::v14_2_0::AioCompletion*, librados::v14_2_0::ObjectWriteOperation*)
     | |   + 0.20% librados::IoCtxImpl::aio_operate(object_t const&, ObjectOperation*, librados::AioCompletionImpl*, SnapContext const&, int, blkin_trace_info const*) [clone .isra.0]
     | |    + 0.20% Objecter::op_submit(Objecter::Op*, unsigned long*, int*)
     | |     + 0.20% Objecter::_op_submit(Objecter::Op*, ceph::shunique_lock<std::shared_mutex>&, unsigned long*)
     | |      + 0.20% std::unique_lock<std::shared_mutex>::lock()
     | |       + 0.20% __pthread_rwlock_wrlock
     | |        + 0.20% __futex_abstimed_wait_common
     | + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::retire_entries(unsigned long) [clone .localalias] [clone .lto_priv.0]
     |  + 0.20% ThreadPool::PointerWQ<Context>::queue(Context*)
     |   + 0.20% std::condition_variable::notify_one()
     |    + 0.20% pthread_cond_signal
     + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
      + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::dispatch_deferred_writes() [clone .localalias] [clone .lto_priv.0]
      |+ 0.20% pthread_mutex_lock
      | + 0.20% __GI___lll_lock_wait
      + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::append_scheduled(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, bool&, bool&, bool) [clone .localalias] [clone .lto_priv.0]
       + 0.20% pthread_mutex_lock
        + 0.20% __GI___lll_lock_wait

Thread 2816 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 98.00% pthread_cond_clockwait
    |+ 97.60% __futex_abstimed_wait_common
    |+ 0.40% __pthread_mutex_cond_lock
    | + 0.40% __GI___lll_lock_wait
    + 1.20% Context::complete(int)
    |+ 0.60% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    ||+ 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::append_scheduled(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, bool&, bool&, bool) [clone .localalias] [clone .lto_priv.0]
    |||+ 0.20% pthread_mutex_lock
    ||| + 0.20% __GI___lll_lock_wait
    ||+ 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::dispatch_deferred_writes() [clone .localalias] [clone .lto_priv.0]
    |||+ 0.20% pthread_mutex_lock
    ||| + 0.20% __GI___lll_lock_wait
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
    || + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
    ||  + 0.20% std::recursive_mutex::lock()
    ||   + 0.20% pthread_mutex_lock
    ||    + 0.20% __GI___lll_lock_wait
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::handle_flushed_sync_point(std::shared_ptr<librbd::cache::pwl::SyncPointLogEntry>)::{lambda(int)#1}>::finish(int)
    ||+ 0.20% pthread_mutex_lock
    || + 0.20% __GI___lll_lock_wait
    |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    || + 0.20% KernelDevice::aio_submit(IOContext*)
    ||  + 0.20% aio_queue_t::submit_batch(std::_List_iterator<aio_t>, std::_List_iterator<aio_t>, unsigned short, void*, int*)
    ||   + 0.20% io_submit
    ||    + 0.20% syscall
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
    | + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
    |  + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
    |   + 0.20% librbd::cache::pwl::ImageCacheState<librbd::ImageCtx>::write_image_cache_state(std::unique_lock<std::mutex>&, Context*) [clone .localalias] [clone .lto_priv.0]
    |    + 0.20% json_spirit::write(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, unsigned int)
    |     + 0.20% json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >::String_type json_spirit::write_string<json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, unsigned int)
    |      + 0.20% json_spirit::Generator<json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> > >::output(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&)
    |       + 0.20% json_spirit::Generator<json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> > >::output(json_spirit::Value_impl<json_spirit::Config_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&)
    |        + 0.20% std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > json_spirit::to_str<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(char const*)
    + 0.60% std::unique_lock<std::mutex>::lock()
    |+ 0.60% pthread_mutex_lock
    | + 0.60% __GI___lll_lock_wait
    + 0.20% ThreadPool::PointerWQ<Context>::_void_dequeue()

Thread 2814 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 98.00% pthread_cond_clockwait
    |+ 98.00% __futex_abstimed_wait_common
    + 1.60% Context::complete(int)
    |+ 1.00% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
    ||+ 0.60% std::recursive_mutex::lock()
    |||+ 0.60% pthread_mutex_lock
    ||| + 0.60% __GI___lll_lock_wait
    ||+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
    || + 0.20% ceph::buffer::v15_2_0::list::obtain_contiguous_space(unsigned int)
    || |+ 0.20% ceph::buffer::v15_2_0::create(unsigned int)
    || | + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
    || |  + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
    || |   + 0.20% ceph::buffer::v15_2_0::raw_combined::create(unsigned int, unsigned int, int)
    || |    + 0.20% mempool::get_pool(mempool::pool_index_t)
    || + 0.20% ceph::buffer::v15_2_0::list::append_zero(unsigned int)
    ||  + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
    ||   + 0.20% posix_memalign
    ||    + 0.20% malloc
    ||     + 0.20% _int_malloc
    |+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
    ||+ 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
    || + 0.40% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
    ||  + 0.20% std::recursive_mutex::lock()
    ||  |+ 0.20% pthread_mutex_lock
    ||  | + 0.20% __GI___lll_lock_wait
    ||  + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::write_log_entries(std::vector<std::shared_ptr<librbd::cache::pwl::GenericLogEntry>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogEntry> > >, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*, unsigned long*) [clone .localalias] [clone .lto_priv.0]
    ||   + 0.20% ceph::buffer::v15_2_0::list::append_zero(unsigned int)
    ||    + 0.20% ceph::buffer::v15_2_0::list::refill_append_space(unsigned int)
    ||     + 0.20% ceph::buffer::v15_2_0::list::get_mempool() const
    |+ 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
    | + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
    |  + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
    |   + 0.20% __GI___pthread_mutex_unlock_usercnt
    |    + 0.20% __GI___lll_lock_wake
    + 0.20% __GI___pthread_mutex_unlock_usercnt
    |+ 0.20% __GI___lll_lock_wake
    + 0.20% ThreadPool::PointerWQ<Context>::_void_dequeue()

Thread 2817 (tp_pwl) - 500 samples
+ 100.00% __clone
 + 100.00% start_thread
  + 100.00% ThreadPool::WorkThread::entry()
   + 100.00% ThreadPool::worker(ThreadPool::WorkThread*)
    + 99.40% pthread_cond_clockwait
    |+ 99.40% __futex_abstimed_wait_common
    + 0.60% Context::complete(int)
     + 0.20% LambdaContext<librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::wake_up()::{lambda(int)#1}>::finish(int)
     |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::process_work() [clone .localalias] [clone .lto_priv.0]
     | + 0.20% librbd::cache::pwl::AbstractWriteLog<librbd::ImageCtx>::process_writeback_dirty_entries() [clone .localalias] [clone .lto_priv.0]
     |  + 0.20% std::recursive_mutex::lock()
     |   + 0.20% pthread_mutex_lock
     |    + 0.20% __GI___lll_lock_wait
     + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_root_scheduled_ops() [clone .localalias]
     |+ 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::update_pool_root(std::shared_ptr<librbd::cache::pwl::WriteLogPoolRoot>, librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::AioTransContext*) [clone .localalias]
     | + 0.20% KernelDevice::aio_write(unsigned long, ceph::buffer::v15_2_0::list&, IOContext*, bool, int)
     |  + 0.20% ceph::buffer::v15_2_0::list::rebuild_aligned_size_and_memory(unsigned int, unsigned int, unsigned int)
     |   + 0.20% ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)
     |    + 0.20% ceph::buffer::v15_2_0::create_aligned_in_mempool(unsigned int, unsigned int, int)
     |     + 0.20% operator new(unsigned long)
     |      + 0.20% malloc
     |       + 0.20% _int_malloc
     + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_scheduled_ops() [clone .localalias] [clone .lto_priv.0]
      + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_op_log_entries(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&) [clone .localalias]
       + 0.20% librbd::cache::pwl::ssd::WriteLog<librbd::ImageCtx>::append_ops(std::__cxx11::list<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >&, Context*, unsigned long*) [clone .localalias]
        + 0.20% std::vector<std::shared_ptr<librbd::cache::pwl::GenericLogOperation>, std::allocator<std::shared_ptr<librbd::cache::pwl::GenericLogOperation> > >::~vector()