Skip to content

Instantly share code, notes, and snippets.

@aclamk

aclamk/gdb.txt Secret

Created July 17, 2020 16:20
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save aclamk/fac791df3510840c640e18a0e6a4c724 to your computer and use it in GitHub Desktop.
Save aclamk/fac791df3510840c640e18a0e6a4c724 to your computer and use it in GitHub Desktop.
Download ZIP
"bench" problem
Raw
gdb.txt
(gdb) print (*(BlueStore::TransContext*)0x56251395a840).oncommit
There is no member or method named oncommit.
(gdb) print (*(BlueStore::TransContext*)0x56251395a840).on_commit
There is no member or method named on_commit.
(gdb) print (*(BlueStore::TransContext*)0x56251395a840).state
$18 = BlueStore::TransContext::STATE_DEFERRED_CLEANUP
(gdb) print (*(BlueStore::TransContext*)0x56251395a840).oncommits
$19 = {<std::_List_base<Context*, std::allocator<Context*> >> = {
_M_impl = {<std::allocator<std::_List_node<Context*> >> = {<__gnu_cxx::new_allocator<std::_List_node<Context*> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0x56251395a950, _M_prev = 0x56251395a950}}}, <No data fields>}
(gdb) print &waiter
$20 = (C_SaferCond *) 0x7f6975db31b0
(gdb) p (*(BlueStore::TransContext*)0x56251395a840).oncommits
$21 = {<std::_List_base<Context*, std::allocator<Context*> >> = {
_M_impl = {<std::allocator<std::_List_node<Context*> >> = {<__gnu_cxx::new_allocator<std::_List_node<Context*> >> = {<No data fields>}, <No data fields>}, _M_node = {_M_next = 0x56251395a950, _M_prev = 0x56251395a950}}}, <No data fields>}
(gdb) p (*(BlueStore::TransContext*)0x56251395a840).oncommits._M_impl._M_node
$22 = {_M_next = 0x56251395a950, _M_prev = 0x56251395a950}
(gdb) p &(*(BlueStore::TransContext*)0x56251395a840).oncommits._M_impl._M_node
$23 = (std::__detail::_List_node_base *) 0x56251395a950
(gdb) info threads
Id Target Id Frame
59 Thread 0x7f69775b9700 (LWP 10423) "log" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
58 Thread 0x7f69755b5700 (LWP 10424) "service" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
* 57 Thread 0x7f6975db6700 (LWP 10425) "admin_socket" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
56 Thread 0x7f6976db8700 (LWP 10458) "msgr-worker-0" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
55 Thread 0x7f69765b7700 (LWP 10459) "msgr-worker-1" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
54 Thread 0x7f69745b3700 (LWP 10460) "msgr-worker-2" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
53 Thread 0x7f6974db4700 (LWP 10461) "signal_handler" 0x00007f6979621a3d in poll ()
at ../sysdeps/unix/syscall-template.S:81
52 Thread 0x7f6973db2700 (LWP 10462) "io_context_pool" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
51 Thread 0x7f69735b1700 (LWP 10463) "io_context_pool" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
50 Thread 0x7f6971d2c700 (LWP 10486) "OpHistorySvc" 0x00007f69795f31ad in nanosleep ()
at ../sysdeps/unix/syscall-template.S:81
49 Thread 0x7f697252d700 (LWP 10487) "ceph_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
48 Thread 0x7f697152b700 (LWP 10488) "safe_timer" __lll_lock_wait ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
47 Thread 0x7f6970d2a700 (LWP 10489) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
46 Thread 0x7f6970529700 (LWP 10490) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
45 Thread 0x7f696fd28700 (LWP 10491) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
44 Thread 0x7f696f527700 (LWP 10492) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
43 Thread 0x7f696ed26700 (LWP 10493) "bstore_aio" 0x00007f697bcd2644 in __io_getevents_0_4 () from /lib64/libaio.so.1
42 Thread 0x7f696e525700 (LWP 10494) "bstore_discard" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
41 Thread 0x7f696cd22700 (LWP 10498) "rocksdb:low0" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
40 Thread 0x7f696c521700 (LWP 10499) "rocksdb:low1" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
39 Thread 0x7f696bd20700 (LWP 10500) "rocksdb:high0" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
38 Thread 0x7f696d523700 (LWP 10622) "bstore_aio" 0x00007f697bcd2644 in __io_getevents_0_4 () from /lib64/libaio.so.1
37 Thread 0x7f696dd24700 (LWP 10623) "bstore_discard" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
36 Thread 0x7f696b51f700 (LWP 10624) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
35 Thread 0x7f6965af7700 (LWP 10745) "rocksdb:dump_st" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
34 Thread 0x7f69662f8700 (LWP 10746) "rocksdb:pst_st" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
---Type <return> to continue, or q <return> to quit---
33 Thread 0x7f6966af9700 (LWP 10747) "cfin" __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
32 Thread 0x7f69672fa700 (LWP 10748) "bstore_kv_sync" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
31 Thread 0x7f696ab01700 (LWP 10749) "bstore_kv_final" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
30 Thread 0x7f696a300700 (LWP 10750) "bstore_mempool" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
29 Thread 0x7f6964eb2700 (LWP 10751) "ms_dispatch" __lll_lock_wait ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
28 Thread 0x7f69646b1700 (LWP 10752) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
27 Thread 0x7f6963eb0700 (LWP 10753) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
26 Thread 0x7f69636af700 (LWP 10754) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
25 Thread 0x7f6962eae700 (LWP 10755) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
24 Thread 0x7f69626ad700 (LWP 10756) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
23 Thread 0x7f6961eac700 (LWP 10757) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
22 Thread 0x7f69616ab700 (LWP 10758) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
21 Thread 0x7f6960eaa700 (LWP 10759) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
20 Thread 0x7f69606a9700 (LWP 10760) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
19 Thread 0x7f695fea8700 (LWP 10761) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
18 Thread 0x7f695f6a7700 (LWP 10762) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
17 Thread 0x7f695eea6700 (LWP 10763) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
16 Thread 0x7f695e6a5700 (LWP 10764) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
15 Thread 0x7f695dea4700 (LWP 10765) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
14 Thread 0x7f695d6a3700 (LWP 10766) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
13 Thread 0x7f695cea2700 (LWP 10767) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
12 Thread 0x7f695c6a1700 (LWP 10768) "finisher" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
11 Thread 0x7f695bea0700 (LWP 10769) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
10 Thread 0x7f695b69f700 (LWP 10770) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
9 Thread 0x7f695ae9e700 (LWP 10771) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
8 Thread 0x7f695a69d700 (LWP 10772) "osd_srv_agent" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
---Type <return> to continue, or q <return> to quit---q
Quit
(gdb) thread 32
[Switching to thread 32 (Thread 0x7f69672fa700 (LWP 10748))]
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
185 62: movl (%rsp), %edi
(gdb) bt
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x00007f6979ec40ec in __gthread_cond_wait (__mutex=<optimized out>,
__cond=__cond@entry=0x7f697b07c31a <tcmalloc::CentralFreeList::FetchFromOneSpansSafe(int, void**, void**)+26>)
at /home/perf/src/adam/gcc-7.2.0/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
#2 std::condition_variable::wait (this=this@entry=0x5624ffe36558, __lock=...)
at ../../../.././libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00005624f45ab191 in BlueStore::_kv_sync_thread (this=0x5624ffe36000)
at /home/akupczyk/ceph-1x/src/os/bluestore/BlueStore.cc:11941
#4 0x00005624f45ce99d in BlueStore::KVSyncThread::entry (this=<optimized out>)
at /home/akupczyk/ceph-1x/src/os/bluestore/BlueStore.h:1945
#5 0x00007f697a5c4e25 in start_thread (arg=0x7f69672fa700) at pthread_create.c:308
#6 0x00007f697962c34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) frame 4
#4 0x00005624f45ce99d in BlueStore::KVSyncThread::entry (this=<optimized out>)
at /home/akupczyk/ceph-1x/src/os/bluestore/BlueStore.h:1945
1945 store->_kv_sync_thread();
(gdb) print store->finisher
value has been optimized out
(gdb) frame 3
#3 0x00005624f45ab191 in BlueStore::_kv_sync_thread (this=0x5624ffe36000)
at /home/akupczyk/ceph-1x/src/os/bluestore/BlueStore.cc:11941
11941 kv_cond.wait(l);
(gdb) print this
$24 = (BlueStore * const) 0x5624ffe36000
(gdb) print finisher
$25 = {cct = 0x5624ff254000, finisher_lock = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 0, __count = 0,
__owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}}, <No data fields>}, finisher_cond = {_M_cond = {__data = {
__lock = 0, __futex = 528, __total_seq = 264, __wakeup_seq = 264, __woken_seq = 264, __mutex = 0x5624ffe36408,
__nwaiters = 0, __broadcast_seq = 264},
__size = "\000\000\000\000\020\002\000\000\b\001\000\000\000\000\000\000\b\001\000\000\000\000\000\000\b\001\000\000\000\000\000\000\bd\343\377$V\000\000\000\000\000\000\b\001\000", __align = 2267742732288}}, finisher_empty_cond = {_M_cond = {
__data = {__lock = 0, __futex = 0, __total_seq = 0, __wakeup_seq = 0, __woken_seq = 0, __mutex = 0x0, __nwaiters = 0,
__broadcast_seq = 0}, __size = '\000' <repeats 47 times>, __align = 0}}, finisher_stop = false,
finisher_running = true, finisher_empty_wait = false,
finisher_queue = {<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {<std::allocator<std::pair<Context*, int> >> = {<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>}, _M_start = 0x5624ff1d9430, _M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440}}, <No data fields>},
in_progress_queue = {<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {<std::allocator<std::pair<Context*, int> >> = {<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>}, _M_start = 0x5624ff1d91f0, _M_finish = 0x5624ff1d9200,
_M_end_of_storage = 0x5624ff1d9200}}, <No data fields>}, thread_name = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
_M_p = 0x5624ffe4da58 "cfin"}}, logger = 0x5624fff02620, finisher_thread = {<Thread> = {
_vptr.Thread = 0x5624f5350af8 <vtable for Finisher::FinisherThread+16>, thread_id = 140090671077120, pid = 10747,
cpuid = -1, thread_name = 0x5624ffe4da58 "cfin"}, fin = 0x5624ffe36400}}
(gdb) print finisher.finisher_queue
$26 = {<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {<std::allocator<std::pair<Context*, int> >> = {<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>}, _M_start = 0x5624ff1d9430, _M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440}}, <No data fields>}
(gdb) p finisher.finisher_queue
$27 = {<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {<std::allocator<std::pair<Context*, int> >> = {<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>}, _M_start = 0x5624ff1d9430, _M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440}}, <No data fields>}
(gdb) set print pretty on
(gdb) p finisher.finisher_queue
$28 = {
<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {
<std::allocator<std::pair<Context*, int> >> = {
<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >::_Vector_impl:
_M_start = 0x5624ff1d9430,
_M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440
}
}, <No data fields>}
(gdb) p finisher.finisher_queue[0]
Could not find operator[].
(gdb) x/20x 0x5624ff1d9430
0x5624ff1d9430: 0x75db31b0 0x00007f69 0x00000000 0x16229044
0x5624ff1d9440: 0xff1d9460 0x00005624 0x00010000 0xeab6a9dc
0x5624ff1d9450: 0xff1d9180 0x00005624 0x00010000 0xeab6a9dc
0x5624ff1d9460: 0xff1d9d90 0x00005624 0x00010000 0xeab6a9dc
0x5624ff1d9470: 0xff1d9480 0x00005624 0xfff17400 0x00005624
(gdb) x/20gx 0x5624ff1d9430
0x5624ff1d9430: 0x00007f6975db31b0 0x1622904400000000
0x5624ff1d9440: 0x00005624ff1d9460 0xeab6a9dc00010000
0x5624ff1d9450: 0x00005624ff1d9180 0xeab6a9dc00010000
0x5624ff1d9460: 0x00005624ff1d9d90 0xeab6a9dc00010000
0x5624ff1d9470: 0x00005624ff1d9480 0x00005624fff17400
0x5624ff1d9480: 0x00005624ff1d9490 0x00005624fff17400
0x5624ff1d9490: 0x00005624ff1d94a0 0x00005624fff17400
0x5624ff1d94a0: 0x00005624ff1d94b0 0x00005624fff17400
0x5624ff1d94b0: 0x00005624ff1d8e40 0x00005624fff17400
0x5624ff1d94c0: 0x0000000000000000 0x000056250dae4600
(gdb) p *(std::pair<Context*, int>*)0x5624ff1d9430
$29 = {
first = 0x7f6975db31b0,
second = 0
}
(gdb) p *(std::pair<Context*, int>*)0x5624ff1d9430->first
Attempt to extract a component of a value that is not a structure pointer.
(gdb) p *(std::pair<Context*, int>*)0x5624ff1d9430.first
Invalid number "0x5624ff1d9430.first".
(gdb) p (*(std::pair<Context*, int>*)0x5624ff1d9430).first
$30 = (Context *) 0x7f6975db31b0
(gdb) p *(*(std::pair<Context*, int>*)0x5624ff1d9430).first
$31 = {
_vptr.Context = 0x5624f5350ab8 <vtable for C_SaferCond+16>
}
(gdb) p *(*(std^CQuit<Context*, int>*)0x5624ff1d9430).first
(gdb) info threads
Id Target Id Frame
59 Thread 0x7f69775b9700 (LWP 10423) "log" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
58 Thread 0x7f69755b5700 (LWP 10424) "service" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
57 Thread 0x7f6975db6700 (LWP 10425) "admin_socket" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
56 Thread 0x7f6976db8700 (LWP 10458) "msgr-worker-0" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
55 Thread 0x7f69765b7700 (LWP 10459) "msgr-worker-1" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
54 Thread 0x7f69745b3700 (LWP 10460) "msgr-worker-2" 0x00007f697962c923 in epoll_wait ()
at ../sysdeps/unix/syscall-template.S:81
53 Thread 0x7f6974db4700 (LWP 10461) "signal_handler" 0x00007f6979621a3d in poll ()
at ../sysdeps/unix/syscall-template.S:81
52 Thread 0x7f6973db2700 (LWP 10462) "io_context_pool" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
51 Thread 0x7f69735b1700 (LWP 10463) "io_context_pool" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
50 Thread 0x7f6971d2c700 (LWP 10486) "OpHistorySvc" 0x00007f69795f31ad in nanosleep ()
at ../sysdeps/unix/syscall-template.S:81
49 Thread 0x7f697252d700 (LWP 10487) "ceph_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
48 Thread 0x7f697152b700 (LWP 10488) "safe_timer" __lll_lock_wait ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
47 Thread 0x7f6970d2a700 (LWP 10489) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
46 Thread 0x7f6970529700 (LWP 10490) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
45 Thread 0x7f696fd28700 (LWP 10491) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
44 Thread 0x7f696f527700 (LWP 10492) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
43 Thread 0x7f696ed26700 (LWP 10493) "bstore_aio" 0x00007f697bcd2644 in __io_getevents_0_4 () from /lib64/libaio.so.1
42 Thread 0x7f696e525700 (LWP 10494) "bstore_discard" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
41 Thread 0x7f696cd22700 (LWP 10498) "rocksdb:low0" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
40 Thread 0x7f696c521700 (LWP 10499) "rocksdb:low1" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
39 Thread 0x7f696bd20700 (LWP 10500) "rocksdb:high0" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
38 Thread 0x7f696d523700 (LWP 10622) "bstore_aio" 0x00007f697bcd2644 in __io_getevents_0_4 () from /lib64/libaio.so.1
37 Thread 0x7f696dd24700 (LWP 10623) "bstore_discard" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
36 Thread 0x7f696b51f700 (LWP 10624) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
35 Thread 0x7f6965af7700 (LWP 10745) "rocksdb:dump_st" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
34 Thread 0x7f69662f8700 (LWP 10746) "rocksdb:pst_st" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
---Type <return> to continue, or q <return> to quit---
33 Thread 0x7f6966af9700 (LWP 10747) "cfin" __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
* 32 Thread 0x7f69672fa700 (LWP 10748) "bstore_kv_sync" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
31 Thread 0x7f696ab01700 (LWP 10749) "bstore_kv_final" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
30 Thread 0x7f696a300700 (LWP 10750) "bstore_mempool" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
29 Thread 0x7f6964eb2700 (LWP 10751) "ms_dispatch" __lll_lock_wait ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
28 Thread 0x7f69646b1700 (LWP 10752) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
27 Thread 0x7f6963eb0700 (LWP 10753) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
26 Thread 0x7f69636af700 (LWP 10754) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
25 Thread 0x7f6962eae700 (LWP 10755) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
24 Thread 0x7f69626ad700 (LWP 10756) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
23 Thread 0x7f6961eac700 (LWP 10757) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
22 Thread 0x7f69616ab700 (LWP 10758) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
21 Thread 0x7f6960eaa700 (LWP 10759) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
20 Thread 0x7f69606a9700 (LWP 10760) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
19 Thread 0x7f695fea8700 (LWP 10761) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
18 Thread 0x7f695f6a7700 (LWP 10762) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
17 Thread 0x7f695eea6700 (LWP 10763) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
16 Thread 0x7f695e6a5700 (LWP 10764) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
15 Thread 0x7f695dea4700 (LWP 10765) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
14 Thread 0x7f695d6a3700 (LWP 10766) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
13 Thread 0x7f695cea2700 (LWP 10767) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
12 Thread 0x7f695c6a1700 (LWP 10768) "finisher" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
11 Thread 0x7f695bea0700 (LWP 10769) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
10 Thread 0x7f695b69f700 (LWP 10770) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
9 Thread 0x7f695ae9e700 (LWP 10771) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
8 Thread 0x7f695a69d700 (LWP 10772) "osd_srv_agent" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
---Type <return> to continue, or q <return> to quit---
7 Thread 0x7f6959e9c700 (LWP 10773) "tp_osd_tp" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
6 Thread 0x7f695969b700 (LWP 10774) "tp_osd_tp" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
5 Thread 0x7f6958e9a700 (LWP 10775) "tp_osd_tp" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
4 Thread 0x7f6958699700 (LWP 10776) "tp_osd_tp" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
3 Thread 0x7f6957e98700 (LWP 10777) "tp_osd_tp" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
2 Thread 0x7f6957697700 (LWP 10778) "osd_srv_heartbt" pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
1 Thread 0x7f697c533bc0 (LWP 10421) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
(gdb) thread 12
[Switching to thread 12 (Thread 0x7f695c6a1700 (LWP 10768))]
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
185 62: movl (%rsp), %edi
(gdb) bt
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x00007f6979ec40ec in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x5624ff22b030)
at /home/perf/src/adam/gcc-7.2.0/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864
#2 std::condition_variable::wait (this=this@entry=0x5624ff22b030, __lock=...)
at ../../../.././libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00005624f46a3b36 in Finisher::finisher_thread_entry (this=0x5624ff22b000)
at /home/akupczyk/ceph-1x/src/common/Finisher.cc:86
#4 0x00007f697a5c4e25 in start_thread (arg=0x7f695c6a1700) at pthread_create.c:308
#5 0x00007f697962c34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) frame 3
#3 0x00005624f46a3b36 in Finisher::finisher_thread_entry (this=0x5624ff22b000)
at /home/akupczyk/ceph-1x/src/common/Finisher.cc:86
86 finisher_cond.wait(ul);
(gdb) list
81 finisher_empty_cond.notify_all();
82 if (finisher_stop)
83 break;
84
85 ldout(cct, 10) << "finisher_thread sleeping" << dendl;
86 finisher_cond.wait(ul);
87 }
88 // If we are exiting, we signal the thread waiting in stop(),
89 // otherwise it would never unblock
90 finisher_empty_cond.notify_all();
(gdb) print this
$32 = (Finisher * const) 0x5624ff22b000
(gdb) print finisher_queue
$33 = {
<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {
<std::allocator<std::pair<Context*, int> >> = {
<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >::_Vector_impl:
_M_start = 0x0,
_M_finish = 0x0,
_M_end_of_storage = 0x0
}
}, <No data fields>}
(gdb) thread 33
[Switching to thread 33 (Thread 0x7f6966af9700 (LWP 10747))]
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
135 2: movl %edx, %eax
(gdb) list
130
131 1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
132 movl $SYS_futex, %eax
133 syscall
134
135 2: movl %edx, %eax
136 xchgl %eax, (%rdi) /* NB: lock is implied */
137
138 testl %eax, %eax
139 jnz 1b
(gdb) bt
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007f697a5c6dcb in _L_lock_812 () from /lib64/libpthread.so.0
#2 0x00007f697a5c6c98 in __GI___pthread_mutex_lock (mutex=0x5624ff256818) at ../nptl/pthread_mutex_lock.c:79
#3 0x00005624f40d4f63 in __gthread_mutex_lock (__mutex=<optimized out>)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/x86_64-redhat-linux/bits/gthr-default.h:748
#4 std::mutex::lock (this=<optimized out>) at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_mutex.h:103
#5 0x00005624f40c4d59 in lock_guard (__m=..., this=<synthetic pointer>)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_mutex.h:162
#6 OSD::_committed_osd_maps (this=0x5624ff256800, first=6, last=6, m=0x5625004c01c0)
at /home/akupczyk/ceph-1x/src/osd/OSD.cc:8087
#7 0x00005624f411e747 in C_OnMapCommit::finish (this=0x56250db25be0, r=<optimized out>)
at /home/akupczyk/ceph-1x/src/osd/OSD.cc:7707
#8 0x00005624f40cda79 in Context::complete (this=0x56250db25be0, r=<optimized out>)
at /home/akupczyk/ceph-1x/src/include/Context.h:99
#9 0x00005624f46a395d in Finisher::finisher_thread_entry (this=0x5624ffe36400)
at /home/akupczyk/ceph-1x/src/common/Finisher.cc:66
#10 0x00007f697a5c4e25 in start_thread (arg=0x7f6966af9700) at pthread_create.c:308
#11 0x00007f697962c34d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113
(gdb) frame 9
#9 0x00005624f46a395d in Finisher::finisher_thread_entry (this=0x5624ffe36400)
at /home/akupczyk/ceph-1x/src/common/Finisher.cc:66
66 p.first->complete(p.second);
(gdb) print finisher_queue
$34 = {
<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {
<std::allocator<std::pair<Context*, int> >> = {
<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >::_Vector_impl:
_M_start = 0x5624ff1d9430,
_M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440
}
}, <No data fields>}
(gdb) fraem 5
Undefined command: "fraem". Try "help".
(gdb) frame 5
#5 0x00005624f40c4d59 in lock_guard (__m=..., this=<synthetic pointer>)
at /opt/rh/devtoolset-7/root/usr/include/c++/7/bits/std_mutex.h:162
162 { _M_device.lock(); }
(gdb) frame 9
#9 0x00005624f46a395d in Finisher::finisher_thread_entry (this=0x5624ffe36400)
at /home/akupczyk/ceph-1x/src/common/Finisher.cc:66
66 p.first->complete(p.second);
(gdb) list
61 count = in_progress_queue.size();
62 }
63
64 // Now actually process the contexts.
65 for (auto p : in_progress_queue) {
66 p.first->complete(p.second);
67 }
68 ldout(cct, 10) << "finisher_thread done with " << in_progress_queue
69 << dendl;
70 in_progress_queue.clear();
(gdb) print in_progress_queue
$35 = {
<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {
<std::allocator<std::pair<Context*, int> >> = {
<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >::_Vector_impl:
_M_start = 0x5624ff1d91f0,
_M_finish = 0x5624ff1d9200,
_M_end_of_storage = 0x5624ff1d9200
}
}, <No data fields>}
(gdb) print finisher_queue
$36 = {
<std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >> = {
_M_impl = {
<std::allocator<std::pair<Context*, int> >> = {
<__gnu_cxx::new_allocator<std::pair<Context*, int> >> = {<No data fields>}, <No data fields>},
members of std::_Vector_base<std::pair<Context*, int>, std::allocator<std::pair<Context*, int> > >::_Vector_impl:
_M_start = 0x5624ff1d9430,
_M_finish = 0x5624ff1d9440,
_M_end_of_storage = 0x5624ff1d9440
}
}, <No data fields>}
(gdb) frame 6
#6 OSD::_committed_osd_maps (this=0x5624ff256800, first=6, last=6, m=0x5625004c01c0)
at /home/akupczyk/ceph-1x/src/osd/OSD.cc:8087
8087 std::lock_guard l(osd_lock);
(gdb) print osd_lock
$37 = {
<std::__mutex_base> = {
_M_mutex = {
__data = {
__lock = 2,
__count = 0,
__owner = 10425,
__nusers = 2,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = "\002\000\000\000\000\000\000\000\271(\000\000\002", '\000' <repeats 26 times>,
__align = 2
}
}, <No data fields>}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment