Skip to content

Instantly share code, notes, and snippets.

@yiwu-arbug
Last active November 13, 2020 20:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save yiwu-arbug/a28d92a05d247b1031325d7259bdff5a to your computer and use it in GitHub Desktop.
Save yiwu-arbug/a28d92a05d247b1031325d7259bdff5a to your computer and use it in GitHub Desktop.
[New LWP 19223]
[New LWP 19194]
[New LWP 19193]
[New LWP 19045]
[New LWP 19044]
[New LWP 19043]
[New LWP 19042]
[New LWP 19041]
[New LWP 19040]
[New LWP 19039]
[New LWP 19038]
[New LWP 19037]
[New LWP 19036]
[New LWP 19035]
[New LWP 19034]
[New LWP 19033]
[New LWP 19032]
[New LWP 19031]
[New LWP 19030]
[New LWP 19029]
[New LWP 19028]
[New LWP 19027]
[New LWP 19026]
[New LWP 19025]
[New LWP 19024]
[New LWP 19023]
[New LWP 19022]
[New LWP 19021]
[New LWP 19020]
[New LWP 19019]
[New LWP 19018]
[New LWP 19017]
[New LWP 19016]
[New LWP 19015]
[New LWP 19014]
[New LWP 19013]
[New LWP 19012]
[New LWP 19011]
[New LWP 19010]
[New LWP 19009]
[New LWP 19008]
[New LWP 19007]
[New LWP 19006]
[New LWP 19005]
[New LWP 19004]
[New LWP 19003]
[New LWP 19002]
[New LWP 19001]
[New LWP 19000]
[New LWP 18999]
[New LWP 18998]
[New LWP 18997]
[New LWP 18996]
[New LWP 18995]
[New LWP 18979]
[New LWP 18978]
[New LWP 18977]
[New LWP 18976]
[New LWP 18975]
[New LWP 18957]
[New LWP 18956]
[New LWP 18955]
[New LWP 18954]
[New LWP 18950]
[New LWP 18943]
[New LWP 18916]
[New LWP 18915]
[New LWP 18914]
[New LWP 18913]
[New LWP 18912]
[New LWP 18911]
[New LWP 18910]
[New LWP 18909]
[New LWP 18908]
[New LWP 18907]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
0x00007fea504e13c1 in sigwait () from /lib64/libpthread.so.0
Thread 76 (Thread 0x7fea4eb7f700 (LWP 18907)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11e99a in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park () at library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at library/std/src/thread/mod.rs:881
#4 std::sync::mpsc::blocking::WaitToken::wait::hae3a13d09b5b5f41 () at library/std/src/sync/mpsc/blocking.rs:64
#5 0x00005637591808be in std::sync::mpsc::Receiver$LT$T$GT$::recv::hb6fa56ea6eb78077 (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/sync.rs:116
#6 0x000056375918be5d in std::sys_common::backtrace::__rust_begin_short_backtrace::h0da9ab3f3e4f06ad (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/slog-async-2.3.0/lib.rs:282
#7 0x000056375918bc8c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h04e12d72cd050c67 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#8 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#10 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 75 (Thread 0x7fea4e3ff700 (LWP 18908)):
#0 0x00007fea504e0e9d in nanosleep () from /lib64/libpthread.so.0
#1 0x000056375a718f45 in std::sys_common::backtrace::__rust_begin_short_backtrace::h5c12cd4542f32383 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5//library/std/src/sys/unix/thread.rs:181
#2 0x000056375a7189b5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h5c37ea1f13ac5791 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#3 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#4 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 74 (Thread 0x7fea4ddff700 (LWP 18909)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375ad20292 in gpr_cv_wait (cv=0x7fea4f2bf038, mu=0x7fea4f2bf000, abs_deadline=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gpr/sync_posix.cc:130
#2 0x000056375ade7b8f in grpc_core::Executor::ThreadMain (arg=0x7fea4f2bf000) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/executor.cc:231
#3 0x000056375ad4c98c in operator() (__closure=0x0, v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:140
#4 grpc_core::(anonymous namespace)::ThreadInternalsPosix::__lambda0::_FUN (v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:145
#5 0x000056375ad4c98c in grpc_core::Thread::Thread (this=<optimized out>, thd_name=<optimized out>, thd_body=<optimized out>, arg=<optimized out>, success=<optimized out>, options=...)
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 73 (Thread 0x7fea4d1fe700 (LWP 18910)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375ad20292 in gpr_cv_wait (cv=0x7fea4f2bf638, mu=0x7fea4f2bf600, abs_deadline=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gpr/sync_posix.cc:130
#2 0x000056375ade7b8f in grpc_core::Executor::ThreadMain (arg=0x7fea4f2bf600) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/executor.cc:231
#3 0x000056375ad4c98c in operator() (__closure=0x0, v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:140
#4 grpc_core::(anonymous namespace)::ThreadInternalsPosix::__lambda0::_FUN (v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:145
#5 0x000056375ad4c98c in grpc_core::Thread::Thread (this=<optimized out>, thd_name=<optimized out>, thd_body=<optimized out>, arg=<optimized out>, success=<optimized out>, options=...)
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 72 (Thread 0x7fea4c7fd700 (LWP 18911)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375ad20292 in gpr_cv_wait (cv=0x56375b8c71e0 <g_cv_wait>, mu=0x56375b8c7220 <_ZL4g_mu>, abs_deadline=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gpr/sync_posix.cc:130
#2 0x000056375adf2a6f in wait_until (next=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:201
#3 timer_main_loop () at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:255
#4 timer_thread (completed_thread_ptr=0x7fea4f2928b0) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:284
#5 0x000056375ad4c98c in operator() (__closure=0x0, v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:140
#6 grpc_core::(anonymous namespace)::ThreadInternalsPosix::__lambda0::_FUN (v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:145
#7 0x000056375ad4c98c in grpc_core::Thread::Thread (this=<optimized out>, thd_name=<optimized out>, thd_body=<optimized out>, arg=<optimized out>, success=<optimized out>, options=...)
#8 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 71 (Thread 0x7fea4bdfc700 (LWP 18912)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea4f2c0e00) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea4b4020c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea4b402000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 70 (Thread 0x7fea4bbfb700 (LWP 18913)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea4f2c1400) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea4aa020c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea4aa02000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 69 (Thread 0x7fea4b3ff700 (LWP 18914)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea4f2c1a00) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea4a2020c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea4a202000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 68 (Thread 0x7fea4b1fe700 (LWP 18915)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea4f3e9000) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea49e020c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea49e02000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 67 (Thread 0x7fea4a9ff700 (LWP 18916)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea49f2e000) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea49a020c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea49a02000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 66 (Thread 0x7fea499ff700 (LWP 18943)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a120f08 in std::sys::unix::condvar::Condvar::wait_timeout::h5ac8ecb229600331 () at library/std/src/sys/unix/condvar.rs:98
#2 0x0000563759a15db3 in wait_timeout (self=0x7fea49201068, mutex=0x7fea49201058, dur=<error reading variable: access outside bounds of object referenced via synthetic pointer>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/condvar.rs:56
#3 wait_timeout<()> (self=0x7fea49201068, guard=..., dur=<error reading variable: access outside bounds of object referenced via synthetic pointer>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/condvar.rs:384
#4 crossbeam_utils::sync::parker::Inner::park::hadf2009d1b3af259 (self=0x7fea49201050, timeout=...) at /rust/registry/src/github.com-1ecc6299db9ec823/crossbeam-utils-0.7.2/src/sync/parker.rs:281
#5 0x000056375a7280e0 in tokio_timer::timer::Timer$LT$T$C$N$GT$::turn::hc01f2de1fbd0c952 (self=0x7fea499da970, max_wait=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/result.rs:388
#6 0x000056375a727554 in tikv_util::timer::start_global_timer::_$u7b$$u7b$closure$u7d$$u7d$::h7b29f6e3e19fe265 () at components/tikv_util/src/timer.rs:95
#7 0x000056375a7271b6 in std::sys_common::backtrace::__rust_begin_short_backtrace::he54b852066d32721 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#8 0x000056375a7271a6 in std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::ha16139be4499ce0e () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#9 0x000056375a727196 in _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h62c004d8112d281f (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#10 0x000056375a727084 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h5e3551ddbe455a50 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 65 (Thread 0x7fea489ff700 (LWP 18950)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa41c85 in rocksdb::InstrumentedCondVar::WaitInternal (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:48
#3 0x000056375aa41d29 in rocksdb::InstrumentedCondVar::Wait (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:41
#4 0x000056375a9724a0 in rocksdb::DBImpl::RunManualCompaction (this=this@entry=0x7fea4ecc3c00, cfd=cfd@entry=0x7fea4ed65300, input_level=input_level@entry=0, output_level=output_level@entry=-2, compact_range_options=..., begin=begin@entry=0x7fea489d9cd0, end=end@entry=0x7fea489d9ce0, exclusive=exclusive@entry=false, disallow_trivial_move=disallow_trivial_move@entry=false, max_file_num_to_ignore=max_file_num_to_ignore@entry=18446744073709551615) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc:1467
#5 0x000056375a972ae4 in rocksdb::DBImpl::CompactRange (this=0x7fea4ecc3c00, options=..., column_family=<optimized out>, begin=0x7fea489d9cd0, end=0x7fea489d9ce0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc:751
#6 0x000056375a87e247 in crocksdb_compact_range_cf_opt (db=<optimized out>, column_family=<optimized out>, opt=<optimized out>, start_key=<optimized out>, start_key_len=<optimized out>, limit_key=<optimized out>, limit_key_len=1) at crocksdb/c.cc:1369
#7 0x00005637595f40ac in raftstore::store::worker::compact::Runner$LT$E$GT$::compact_range_cf::hfa77c5a29a15ffa3 (self=<optimized out>, cf_name=..., start_key=..., end_key=...) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/src/rocksdb.rs:1336
#8 0x00005637595f3484 in _$LT$raftstore..store..worker..compact..Runner$LT$E$GT$$u20$as$u20$tikv_util..worker..pool..Runnable$GT$::run::h96b9ff6513648e11 (self=<optimized out>, task=...) at components/raftstore/src/store/worker/compact.rs:157
#9 0x000056375993b6a8 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h83f2773cbec6eb76 (self=..., cx=<optimized out>) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/raftstore/src/store/worker/cleanup.rs:63
#10 0x000056375a7b7a41 in _$LT$yatp..task..future..Runner$u20$as$u20$yatp..pool..runner..Runner$GT$::handle::h8a1a4b26047f59e6 (self=0x7fea489da928, local=<optimized out>, task_cell=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/task/future.rs:261
#11 0x00005637597ab3d0 in std::sys_common::backtrace::__rust_begin_short_backtrace::h88f1cd3afc2f53b4 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/tikv_util/src/yatp_pool/mod.rs:93
#12 0x00005637597ab0fc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6fae60f33055b7cf () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#13 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#14 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#15 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#16 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#17 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 64 (Thread 0x7fea475ff700 (LWP 18954)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375abd0dcc in std::condition_variable::wait(std::unique_lock<std::mutex>&) ()
#2 0x000056375ac866f3 in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x7fea4f216ea0, thread_id=thread_id@entry=0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:197
#3 0x000056375ac86a9e in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x7fea4f23d8f0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:307
#4 0x000056375abd2e90 in execute_native_thread_routine ()
#5 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 63 (Thread 0x7fea467ff700 (LWP 18955)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11e99a in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park () at library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at library/std/src/thread/mod.rs:881
#4 std::sync::mpsc::blocking::WaitToken::wait::hae3a13d09b5b5f41 () at library/std/src/sync/mpsc/blocking.rs:64
#5 0x000056375a323fc0 in rocksdb::sst_partitioner::sst_partitioner_factory_create_partitioner::ha9a189958e3f3bfd (ctx=0x7fea4f3f8150, context=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/oneshot.rs:140
#6 0x000056375a881b60 in crocksdb_sst_partitioner_factory_impl_t::CreatePartitioner (this=<optimized out>, partitioner_context=...) at crocksdb/c.cc:5917
#7 0x000056375ac410fb in rocksdb::Compaction::CreateSstPartitioner (this=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/compaction/compaction.cc:554
#8 0x000056375ac567d8 in rocksdb::CompactionJob::ProcessKeyValueCompaction (this=this@entry=0x7fea467d9a40, sub_compact=0x7fea44c72480) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/compaction/compaction_job.cc:906
#9 0x000056375ac57e00 in rocksdb::CompactionJob::Run (this=this@entry=0x7fea467d9a40) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/compaction/compaction_job.cc:591
#10 0x000056375a96ecd1 in rocksdb::DBImpl::BackgroundCompaction (this=this@entry=0x7fea4ecc3c00, made_progress=made_progress@entry=0x7fea467d9eae, job_context=job_context@entry=0x7fea467d9ed0, log_buffer=log_buffer@entry=0x7fea467da0a0, prepicked_compaction=prepicked_compaction@entry=0x7fea49f824a0, thread_pri=thread_pri@entry=rocksdb::Env::LOW) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc:2759
#11 0x000056375a973762 in rocksdb::DBImpl::BackgroundCallCompaction (this=this@entry=0x7fea4ecc3c00, prepicked_compaction=prepicked_compaction@entry=0x7fea49f824a0, bg_thread_pri=bg_thread_pri@entry=rocksdb::Env::LOW) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc:2317
#12 0x000056375a973c5a in rocksdb::DBImpl::BGWorkCompaction (arg=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_compaction_flush.cc:2092
#13 0x000056375ac868aa in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x7fea4f216ea0, thread_id=thread_id@entry=1) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:266
#14 0x000056375ac86a9e in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x7fea4f23d900) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:307
#15 0x000056375abd2e90 in execute_native_thread_routine ()
#16 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#17 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 62 (Thread 0x7fea45bfe700 (LWP 18956)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375abd0dcc in std::condition_variable::wait(std::unique_lock<std::mutex>&) ()
#2 0x000056375ac866f3 in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x7fea4f216f80, thread_id=thread_id@entry=0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:197
#3 0x000056375ac86a9e in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x7fea4f23d920) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:307
#4 0x000056375abd2e90 in execute_native_thread_routine ()
#5 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 61 (Thread 0x7fea44bff700 (LWP 18957)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4f2bb1c0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa41c85 in rocksdb::InstrumentedCondVar::WaitInternal (this=this@entry=0x7fea4f2bb1c0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:48
#3 0x000056375aa41d29 in rocksdb::InstrumentedCondVar::Wait (this=this@entry=0x7fea4f2bb1c0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:41
#4 0x000056375aa2ab9a in rocksdb::DeleteScheduler::BackgroundEmptyTrash (this=0x7fea4f2bb0d0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/file/delete_scheduler.cc:194
#5 0x000056375abd2e90 in execute_native_thread_routine ()
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 60 (Thread 0x7fea2fff4700 (LWP 18975)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada5e2 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x7fea4f217380, abs_time_us=abs_time_us@entry=1605248247267680) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:120
#2 0x000056375aa41e05 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x7fea4f217380, abs_time_us=abs_time_us@entry=1605248247267680) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:66
#3 0x000056375aa41eb1 in rocksdb::InstrumentedCondVar::TimedWait (this=this@entry=0x7fea4f217380, abs_time_us=abs_time_us@entry=1605248247267680) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:55
#4 0x000056375a926a13 in wait (delay=<optimized out>, this=0x7fea4f217300) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:90
#5 thread (this=0x7fea4f217300) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:117
#6 operator() (__closure=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:38
#7 _M_invoke<> (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1732
#8 operator() (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1720
#9 std::thread::_Impl<std::_Bind_simple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::string const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1} ()> >::_M_run() (this=<optimized out>) at /usr/include/c++/4.8.2/thread:115
#10 0x000056375abd2e90 in execute_native_thread_routine ()
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 59 (Thread 0x7fea309f5700 (LWP 18976)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada5e2 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x7fea4f2172a0, abs_time_us=abs_time_us@entry=1605248247185001) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:120
#2 0x000056375aa41e05 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x7fea4f2172a0, abs_time_us=abs_time_us@entry=1605248247185001) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:66
#3 0x000056375aa41eb1 in rocksdb::InstrumentedCondVar::TimedWait (this=this@entry=0x7fea4f2172a0, abs_time_us=abs_time_us@entry=1605248247185001) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:55
#4 0x000056375a926a13 in wait (delay=<optimized out>, this=0x7fea4f217220) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:90
#5 thread (this=0x7fea4f217220) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:117
#6 operator() (__closure=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:38
#7 _M_invoke<> (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1732
#8 operator() (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1720
#9 std::thread::_Impl<std::_Bind_simple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::string const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1} ()> >::_M_run() (this=<optimized out>) at /usr/include/c++/4.8.2/thread:115
#10 0x000056375abd2e90 in execute_native_thread_routine ()
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 58 (Thread 0x7fea313f6700 (LWP 18977)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375abd0dcc in std::condition_variable::wait(std::unique_lock<std::mutex>&) ()
#2 0x000056375ac866f3 in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x7fea4f216f80, thread_id=thread_id@entry=1) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:197
#3 0x000056375ac86a9e in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x7fea4f23dc00) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/threadpool_imp.cc:307
#4 0x000056375abd2e90 in execute_native_thread_routine ()
#5 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 57 (Thread 0x7fea42fff700 (LWP 18978)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4f2bbac0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa41c85 in rocksdb::InstrumentedCondVar::WaitInternal (this=this@entry=0x7fea4f2bbac0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:48
#3 0x000056375aa41d29 in rocksdb::InstrumentedCondVar::Wait (this=this@entry=0x7fea4f2bbac0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:41
#4 0x000056375aa2ab9a in rocksdb::DeleteScheduler::BackgroundEmptyTrash (this=0x7fea4f2bb9d0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/file/delete_scheduler.cc:194
#5 0x000056375abd2e90 in execute_native_thread_routine ()
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 56 (Thread 0x7fea41fff700 (LWP 18979)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375ad2030f in gpr_cv_wait (cv=0x56375b8c71e0 <g_cv_wait>, mu=0x56375b8c7220 <_ZL4g_mu>, abs_deadline=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gpr/sync_posix.cc:146
#2 0x000056375adf2a6f in wait_until (next=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:201
#3 timer_main_loop () at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:255
#4 timer_thread (completed_thread_ptr=0x7fea4b4462c0) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/timer_manager.cc:284
#5 0x000056375ad4c98c in operator() (__closure=0x0, v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:140
#6 grpc_core::(anonymous namespace)::ThreadInternalsPosix::__lambda0::_FUN (v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:145
#7 0x000056375ad4c98c in grpc_core::Thread::Thread (this=<optimized out>, thd_name=<optimized out>, thd_body=<optimized out>, arg=<optimized out>, success=<optimized out>, options=...)
#8 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 55 (Thread 0x7fea2c3ee700 (LWP 18995)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada5e2 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x7fea4f217620, abs_time_us=abs_time_us@entry=1605248247408962) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:120
#2 0x000056375aa41e05 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x7fea4f217620, abs_time_us=abs_time_us@entry=1605248247408962) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:66
#3 0x000056375aa41eb1 in rocksdb::InstrumentedCondVar::TimedWait (this=this@entry=0x7fea4f217620, abs_time_us=abs_time_us@entry=1605248247408962) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:55
#4 0x000056375a926a13 in wait (delay=<optimized out>, this=0x7fea4f2175a0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:90
#5 thread (this=0x7fea4f2175a0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:117
#6 operator() (__closure=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:38
#7 _M_invoke<> (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1732
#8 operator() (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1720
#9 std::thread::_Impl<std::_Bind_simple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::string const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1} ()> >::_M_run() (this=<optimized out>) at /usr/include/c++/4.8.2/thread:115
#10 0x000056375abd2e90 in execute_native_thread_routine ()
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 54 (Thread 0x7fea2cdef700 (LWP 18996)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada5e2 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x7fea4f217540, abs_time_us=abs_time_us@entry=1605248247246061) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:120
#2 0x000056375aa41e05 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x7fea4f217540, abs_time_us=abs_time_us@entry=1605248247246061) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:66
#3 0x000056375aa41eb1 in rocksdb::InstrumentedCondVar::TimedWait (this=this@entry=0x7fea4f217540, abs_time_us=abs_time_us@entry=1605248247246061) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:55
#4 0x000056375a926a13 in wait (delay=<optimized out>, this=0x7fea4f2174c0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:90
#5 thread (this=0x7fea4f2174c0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:117
#6 operator() (__closure=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/util/repeatable_thread.h:38
#7 _M_invoke<> (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1732
#8 operator() (this=<optimized out>) at /usr/include/c++/4.8.2/functional:1720
#9 std::thread::_Impl<std::_Bind_simple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::string const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1} ()> >::_M_run() (this=<optimized out>) at /usr/include/c++/4.8.2/thread:115
#10 0x000056375abd2e90 in execute_native_thread_routine ()
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 53 (Thread 0x7fea425fe700 (LWP 18997)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea49e08010) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x00005637593cc670 in _$LT$tokio..park..either..Either$LT$A$C$B$GT$$u20$as$u20$tokio..park..Park$GT$::park::h013c3981d499a7f7 (self=0x7fea425d8fa0) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x00005637599e1be2 in std::sys_common::backtrace::__rust_begin_short_backtrace::h1c7d8d64e8f65258 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/basic_scheduler.rs:158
#4 0x00005637599e0c5c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h9f29f708a68cf4dc () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#5 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#8 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 52 (Thread 0x7fea369ff700 (LWP 18998)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a5edb49 in std::sys_common::backtrace::__rust_begin_short_backtrace::he3996a094eec9b61 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x000056375a5ed52c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h13bc79f53af13a2a () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#3 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#4 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 51 (Thread 0x7fea367fe700 (LWP 18999)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597ab3b4 in std::sys_common::backtrace::__rust_begin_short_backtrace::h88f1cd3afc2f53b4 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597ab0fc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6fae60f33055b7cf () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 50 (Thread 0x7fea2d7f0700 (LWP 19000)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00005637597fe5ec in raftstore::store::worker::read::LocalReader$LT$C$C$E$GT$::propose_raft_command::hbce04f5eb9a6145b (self=0x7fea48bb60b0, read_id=..., req=..., cb=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 0x00005637597fd14c in _$LT$tikv..server..raftkv..RaftKv$LT$S$GT$$u20$as$u20$tikv..storage..kv..Engine$GT$::async_snapshot::h1c30e687e6cfd0bb (self=0x7fea48bb6000, ctx=..., cb=...) at components/raftstore/src/store/worker/read.rs:511
#5 0x0000563759953f0e in tikv::storage::kv::snapshot::h4663c3cff7c5d629 (engine=0x7fea48bb6000, ctx=...) at src/storage/kv/mod.rs:435
#6 0x0000563759993e5c in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h1bb6334a8a6cb65f (self=..., cx=0x7fea2d7cb770) at src/storage/mod.rs:233
#7 0x0000563759993486 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h971cc010ae3f17b3 (self=..., cx=0x7fea2d7cb770) at src/read_pool.rs:111
#8 0x000056375a7b7a41 in _$LT$yatp..task..future..Runner$u20$as$u20$yatp..pool..runner..Runner$GT$::handle::h8a1a4b26047f59e6 (self=0x7fea2d7cb870, local=<optimized out>, task_cell=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/task/future.rs:261
#9 0x00005637597a975c in std::sys_common::backtrace::__rust_begin_short_backtrace::h7c5ce84db7c12725 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/tikv_util/src/yatp_pool/mod.rs:93
#10 0x00005637597a940c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6137b24398374930 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 49 (Thread 0x7fea365fd700 (LWP 19001)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00005637597fe5ec in raftstore::store::worker::read::LocalReader$LT$C$C$E$GT$::propose_raft_command::hbce04f5eb9a6145b (self=0x7fea47e080b0, read_id=..., req=..., cb=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 0x00005637597fd14c in _$LT$tikv..server..raftkv..RaftKv$LT$S$GT$$u20$as$u20$tikv..storage..kv..Engine$GT$::async_snapshot::h1c30e687e6cfd0bb (self=0x7fea47e08000, ctx=..., cb=...) at components/raftstore/src/store/worker/read.rs:511
#5 0x0000563759953f0e in tikv::storage::kv::snapshot::h4663c3cff7c5d629 (engine=0x7fea47e08000, ctx=...) at src/storage/kv/mod.rs:435
#6 0x0000563759993e5c in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h1bb6334a8a6cb65f (self=..., cx=0x7fea365d8770) at src/storage/mod.rs:233
#7 0x0000563759993486 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h971cc010ae3f17b3 (self=..., cx=0x7fea365d8770) at src/read_pool.rs:111
#8 0x000056375a7b7a41 in _$LT$yatp..task..future..Runner$u20$as$u20$yatp..pool..runner..Runner$GT$::handle::h8a1a4b26047f59e6 (self=0x7fea365d8870, local=<optimized out>, task_cell=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/task/future.rs:261
#9 0x00005637597a975c in std::sys_common::backtrace::__rust_begin_short_backtrace::h7c5ce84db7c12725 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/tikv_util/src/yatp_pool/mod.rs:93
#10 0x00005637597a940c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6137b24398374930 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 48 (Thread 0x7fea35bfc700 (LWP 19002)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00005637597fe5ec in raftstore::store::worker::read::LocalReader$LT$C$C$E$GT$::propose_raft_command::hbce04f5eb9a6145b (self=0x7fea45c090b0, read_id=..., req=..., cb=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 0x00005637597fd14c in _$LT$tikv..server..raftkv..RaftKv$LT$S$GT$$u20$as$u20$tikv..storage..kv..Engine$GT$::async_snapshot::h1c30e687e6cfd0bb (self=0x7fea45c09000, ctx=..., cb=...) at components/raftstore/src/store/worker/read.rs:511
#5 0x0000563759953f0e in tikv::storage::kv::snapshot::h4663c3cff7c5d629 (engine=0x7fea45c09000, ctx=...) at src/storage/kv/mod.rs:435
#6 0x0000563759993e5c in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h1bb6334a8a6cb65f (self=..., cx=0x7fea35bd7770) at src/storage/mod.rs:233
#7 0x0000563759993486 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h971cc010ae3f17b3 (self=..., cx=0x7fea35bd7770) at src/read_pool.rs:111
#8 0x000056375a7b7a41 in _$LT$yatp..task..future..Runner$u20$as$u20$yatp..pool..runner..Runner$GT$::handle::h8a1a4b26047f59e6 (self=0x7fea35bd7870, local=<optimized out>, task_cell=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/task/future.rs:261
#9 0x00005637597a975c in std::sys_common::backtrace::__rust_begin_short_backtrace::h7c5ce84db7c12725 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/tikv_util/src/yatp_pool/mod.rs:93
#10 0x00005637597a940c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6137b24398374930 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 47 (Thread 0x7fea351fb700 (LWP 19003)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00005637597fe5ec in raftstore::store::worker::read::LocalReader$LT$C$C$E$GT$::propose_raft_command::hbce04f5eb9a6145b (self=0x7fea44c0a0b0, read_id=..., req=..., cb=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 0x00005637597fd14c in _$LT$tikv..server..raftkv..RaftKv$LT$S$GT$$u20$as$u20$tikv..storage..kv..Engine$GT$::async_snapshot::h1c30e687e6cfd0bb (self=0x7fea44c0a000, ctx=..., cb=...) at components/raftstore/src/store/worker/read.rs:511
#5 0x0000563759953f0e in tikv::storage::kv::snapshot::h4663c3cff7c5d629 (engine=0x7fea44c0a000, ctx=...) at src/storage/kv/mod.rs:435
#6 0x000056375997b960 in tikv::coprocessor::endpoint::Endpoint$LT$E$GT$::async_snapshot::he9ffa7ef8efc5daf (engine=0x7fea44c0a000, ctx=<optimized out>) at src/coprocessor/endpoint.rs:379
#7 0x000056375997a587 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h92f1f35bf5b40804 (self=..., cx=0x7fea351d6770) at src/coprocessor/endpoint.rs:400
#8 0x000056375997a326 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h74d5c05b90dda503 (self=..., cx=0x7fea351d6770) at src/read_pool.rs:111
#9 0x000056375a7b7a41 in _$LT$yatp..task..future..Runner$u20$as$u20$yatp..pool..runner..Runner$GT$::handle::h8a1a4b26047f59e6 (self=0x7fea351d6870, local=<optimized out>, task_cell=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/task/future.rs:261
#10 0x00005637597a975c in std::sys_common::backtrace::__rust_begin_short_backtrace::h7c5ce84db7c12725 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/tikv_util/src/yatp_pool/mod.rs:93
#11 0x00005637597a940c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6137b24398374930 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#12 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#14 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#15 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#16 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 46 (Thread 0x7fea347fa700 (LWP 19004)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4f39d6d0) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a7549db in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea347d5810, core=0x7fea4f39d500, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 45 (Thread 0x7fea315f9700 (LWP 19005)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 44 (Thread 0x7fea065ed700 (LWP 19006)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 43 (Thread 0x7fea063ec700 (LWP 19007)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 42 (Thread 0x7fea061eb700 (LWP 19008)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 41 (Thread 0x7fea05fea700 (LWP 19009)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 40 (Thread 0x7fea05de9700 (LWP 19010)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597a7074 in std::sys_common::backtrace::__rust_begin_short_backtrace::h14767851e02bcad9 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597a6dbc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6da5662589bfc3e8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 39 (Thread 0x7fea05be8700 (LWP 19011)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4f39da10) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a7549db in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea05bc3810, core=0x7fea4f39db00, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 38 (Thread 0x7fea059e7700 (LWP 19012)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x00005637597a3529 in yatp::pool::spawn::Local$LT$T$GT$::pop_or_sleep::h24ca9fdb35c830a0 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x00005637597ab3b4 in std::sys_common::backtrace::__rust_begin_short_backtrace::h88f1cd3afc2f53b4 (f=...) at /rust/git/checkouts/yatp-e704b73c3ee279b6/6bbea16/src/pool/worker.rs:36
#3 0x00005637597ab0fc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6fae60f33055b7cf () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#7 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 37 (Thread 0x7fea057e6700 (LWP 19013)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a7b71e9 in std::sys_common::backtrace::__rust_begin_short_backtrace::h446df8ea9687a501 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x000056375a7b6ca0 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h1e96d6300bd660f4 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#3 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#4 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 36 (Thread 0x7fea055e5700 (LWP 19014)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a7b71e9 in std::sys_common::backtrace::__rust_begin_short_backtrace::h446df8ea9687a501 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x000056375a7b6ca0 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h1e96d6300bd660f4 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#3 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#4 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 35 (Thread 0x7fea053e4700 (LWP 19015)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a7b71e9 in std::sys_common::backtrace::__rust_begin_short_backtrace::h446df8ea9687a501 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/parking_lot_core-0.8.0/src/thread_parker/linux.rs:112
#2 0x000056375a7b6ca0 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h1e96d6300bd660f4 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#3 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#4 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#5 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 34 (Thread 0x7fea051e3700 (LWP 19016)):
#0 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a120f08 in std::sys::unix::condvar::Condvar::wait_timeout::h5ac8ecb229600331 () at library/std/src/sys/unix/condvar.rs:98
#2 0x0000563759a15db3 in wait_timeout (self=0x7fea48bc6068, mutex=0x7fea48bc6058, dur=<error reading variable: access outside bounds of object referenced via synthetic pointer>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/condvar.rs:56
#3 wait_timeout<()> (self=0x7fea48bc6068, guard=..., dur=<error reading variable: access outside bounds of object referenced via synthetic pointer>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/condvar.rs:384
#4 crossbeam_utils::sync::parker::Inner::park::hadf2009d1b3af259 (self=0x7fea48bc6050, timeout=...) at /rust/registry/src/github.com-1ecc6299db9ec823/crossbeam-utils-0.7.2/src/sync/parker.rs:281
#5 0x000056375a7280e0 in tokio_timer::timer::Timer$LT$T$C$N$GT$::turn::hc01f2de1fbd0c952 (self=0x7fea051be970, max_wait=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/result.rs:388
#6 0x000056375a72a783 in tikv_util::timer::start_global_steady_timer::_$u7b$$u7b$closure$u7d$$u7d$::hc5daafb809dec86f () at components/tikv_util/src/timer.rs:193
#7 0x000056375a72a4ec in std::sys_common::backtrace::__rust_begin_short_backtrace::hbc2aa75a16bf0e75 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#8 0x000056375a72a4cc in std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h6bb39d8432f3eab4 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#9 0x000056375a72a4ac in _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h8ad1d1da6d360829 (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#10 0x000056375a72a38c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h24fb97b6c30fa810 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 33 (Thread 0x7fea04fe2700 (LWP 19017)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4ecc4758) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa12d29 in rocksdb::WriteThread::LinkOne (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea04fb8d00, newest_writer=newest_writer@entry=0x7fea4ecc4618) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:269
#3 0x000056375aa13199 in rocksdb::WriteThread::JoinBatchGroup (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea04fb8d00) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:396
#4 0x000056375a955fe5 in rocksdb::DBImpl::MultiBatchWriteImpl(rocksdb::WriteOptions const&, std::vector<rocksdb::WriteBatch*, std::allocator<rocksdb::WriteBatch*> >&&, rocksdb::WriteCallback*, unsigned long*, unsigned long, unsigned long*) (this=this@entry=0x7fea4ecc3c00, write_options=..., my_batch=my_batch@entry=<unknown type in /home/tidb/deploy/tikv-20160/bin/tikv-server, CU 0x6031e45, DIE 0x6112cdd>, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, seq_used=seq_used@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:86
#5 0x000056375a95a3a0 in rocksdb::DBImpl::WriteImpl (this=0x7fea4ecc3c00, write_options=..., my_batch=0x7fe9fd652f30, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, disable_memtable=disable_memtable@entry=false, seq_used=seq_used@entry=0x0, batch_cnt=batch_cnt@entry=0, pre_release_callback=pre_release_callback@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:326
#6 0x000056375a95bf32 in rocksdb::DBImpl::Write (this=<optimized out>, write_options=..., my_batch=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:58
#7 0x000056375a87ddea in crocksdb_write (db=<optimized out>, options=<optimized out>, batch=<optimized out>, errptr=0x7fea04fb98d0) at crocksdb/c.cc:1040
#8 0x000056375991a81a in raftstore::store::fsm::peer::PeerFsmDelegate$LT$EK$C$ER$C$T$GT$::destroy_peer::hedf7db0a94504ae9 (self=0x7fea04fbaa58, merged_by_target=false) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/src/rocksdb.rs:790
#9 0x0000563759905bd7 in raftstore::store::fsm::peer::PeerFsmDelegate$LT$EK$C$ER$C$T$GT$::on_raft_message::h41f994b086cd7f9b (self=<optimized out>, msg=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/raftstore/src/store/fsm/peer.rs:1789
#10 0x00005637598ebead in raftstore::store::fsm::peer::PeerFsmDelegate$LT$EK$C$ER$C$T$GT$::handle_msgs::hc69f9d65c8d74aaf (self=0x7fea04fbaa58, msgs=<optimized out>) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/raftstore/src/store/fsm/peer.rs:479
#11 0x00005637598bc045 in _$LT$raftstore..store..fsm..store..RaftPoller$LT$EK$C$ER$C$T$GT$$u20$as$u20$batch_system..batch..PollHandler$LT$raftstore..store..fsm..peer..PeerFsm$LT$EK$C$ER$GT$$C$raftstore..store..fsm..store..StoreFsm$LT$EK$GT$$GT$$GT$::handle_normal::h0dc1485cb8dfd143 (self=<optimized out>, peer=<optimized out>) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/raftstore/src/store/fsm/store.rs:839
#12 0x00005637598a9750 in batch_system::batch::Poller$LT$N$C$C$C$Handler$GT$::poll::hde94178e4a3ee359 (self=<optimized out>) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/batch-system/src/batch.rs:294
#13 0x00005637598a6154 in std::sys_common::backtrace::__rust_begin_short_backtrace::h3405db0626ac42c2 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/batch-system/src/batch.rs:396
#14 0x00005637598a5ffc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7678af8be357f952 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#15 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#16 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#18 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#19 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 32 (Thread 0x7fea035e1700 (LWP 19018)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4ecc4758) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa12d29 in rocksdb::WriteThread::LinkOne (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea035b9740, newest_writer=newest_writer@entry=0x7fea4ecc4618) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:269
#3 0x000056375aa13199 in rocksdb::WriteThread::JoinBatchGroup (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea035b9740) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:396
#4 0x000056375a955fe5 in rocksdb::DBImpl::MultiBatchWriteImpl(rocksdb::WriteOptions const&, std::vector<rocksdb::WriteBatch*, std::allocator<rocksdb::WriteBatch*> >&&, rocksdb::WriteCallback*, unsigned long*, unsigned long, unsigned long*) (this=this@entry=0x7fea4ecc3c00, write_options=..., my_batch=my_batch@entry=<unknown type in /home/tidb/deploy/tikv-20160/bin/tikv-server, CU 0x6031e45, DIE 0x6112cdd>, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, seq_used=seq_used@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:86
#5 0x000056375a95a3a0 in rocksdb::DBImpl::WriteImpl (this=0x7fea4ecc3c00, write_options=..., my_batch=0x7fea4357e900, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, disable_memtable=disable_memtable@entry=false, seq_used=seq_used@entry=0x0, batch_cnt=batch_cnt@entry=0, pre_release_callback=pre_release_callback@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:326
#6 0x000056375a95bf32 in rocksdb::DBImpl::Write (this=<optimized out>, write_options=..., my_batch=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:58
#7 0x000056375a87ddea in crocksdb_write (db=<optimized out>, options=<optimized out>, batch=<optimized out>, errptr=0x7fea035ba230) at crocksdb/c.cc:1040
#8 0x00005637598a9d89 in batch_system::batch::Poller$LT$N$C$C$C$Handler$GT$::poll::hde94178e4a3ee359 (self=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/src/rocksdb.rs:790
#9 0x00005637598a6154 in std::sys_common::backtrace::__rust_begin_short_backtrace::h3405db0626ac42c2 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/batch-system/src/batch.rs:396
#10 0x00005637598a5ffc in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7678af8be357f952 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#11 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#12 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#14 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 31 (Thread 0x7fea033e0700 (LWP 19019)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4ecc4758) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa12d29 in rocksdb::WriteThread::LinkOne (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea033ba430, newest_writer=newest_writer@entry=0x7fea4ecc4618) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:269
#3 0x000056375aa13199 in rocksdb::WriteThread::JoinBatchGroup (this=this@entry=0x7fea4ecc4580, w=w@entry=0x7fea033ba430) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/write_thread.cc:396
#4 0x000056375a955fe5 in rocksdb::DBImpl::MultiBatchWriteImpl(rocksdb::WriteOptions const&, std::vector<rocksdb::WriteBatch*, std::allocator<rocksdb::WriteBatch*> >&&, rocksdb::WriteCallback*, unsigned long*, unsigned long, unsigned long*) (this=this@entry=0x7fea4ecc3c00, write_options=..., my_batch=my_batch@entry=<unknown type in /home/tidb/deploy/tikv-20160/bin/tikv-server, CU 0x6031e45, DIE 0x6112cdd>, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, seq_used=seq_used@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:86
#5 0x000056375a95a3a0 in rocksdb::DBImpl::WriteImpl (this=0x7fea4ecc3c00, write_options=..., my_batch=0x7fea44c44000, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, disable_memtable=disable_memtable@entry=false, seq_used=seq_used@entry=0x0, batch_cnt=batch_cnt@entry=0, pre_release_callback=pre_release_callback@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:326
#6 0x000056375a95bf32 in rocksdb::DBImpl::Write (this=<optimized out>, write_options=..., my_batch=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:58
#7 0x000056375a87ddea in crocksdb_write (db=<optimized out>, options=<optimized out>, batch=<optimized out>, errptr=0x7fea033bada0) at crocksdb/c.cc:1040
#8 0x00005637594b9595 in raftstore::store::fsm::apply::ApplyContext$LT$EK$C$W$GT$::write_to_db::h0da768c7781471b0 (self=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/src/rocksdb.rs:790
#9 0x00005637594b8341 in raftstore::store::fsm::apply::ApplyContext$LT$EK$C$W$GT$::flush::hfbf6667b7288e6f4 (self=0x7fea033bb220) at components/raftstore/src/store/fsm/apply.rs:553
#10 0x00005637594b60c5 in std::sys_common::backtrace::__rust_begin_short_backtrace::h23596ebe92d23460 (f=...) at components/raftstore/src/store/fsm/apply.rs:3540
#11 0x00005637594b581c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hae5a836fa8f5db22 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#12 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#13 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#14 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#15 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#16 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 30 (Thread 0x7fea031df700 (LWP 19020)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375aada57d in rocksdb::port::CondVar::Wait (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/port/port_posix.cc:106
#2 0x000056375aa41c85 in rocksdb::InstrumentedCondVar::WaitInternal (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:48
#3 0x000056375aa41d29 in rocksdb::InstrumentedCondVar::Wait (this=this@entry=0x7fea4ecc4270) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/monitoring/instrumented_mutex.cc:41
#4 0x000056375a951052 in rocksdb::DBImpl::DelayWrite (this=this@entry=0x7fea4ecc3c00, num_bytes=<optimized out>, write_options=...) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:1594
#5 0x000056375a955a98 in rocksdb::DBImpl::PreprocessWrite (this=this@entry=0x7fea4ecc3c00, write_options=..., need_log_sync=need_log_sync@entry=0x7fea031b9330, write_context=write_context@entry=0x7fea031b9530) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:1059
#6 0x000056375a956373 in rocksdb::DBImpl::MultiBatchWriteImpl(rocksdb::WriteOptions const&, std::vector<rocksdb::WriteBatch*, std::allocator<rocksdb::WriteBatch*> >&&, rocksdb::WriteCallback*, unsigned long*, unsigned long, unsigned long*) (this=this@entry=0x7fea4ecc3c00, write_options=..., my_batch=my_batch@entry=<unknown type in /home/tidb/deploy/tikv-20160/bin/tikv-server, CU 0x6031e45, DIE 0x6112cdd>, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, seq_used=seq_used@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:99
#7 0x000056375a95a3a0 in rocksdb::DBImpl::WriteImpl (this=0x7fea4ecc3c00, write_options=..., my_batch=0x7fea43c30000, callback=callback@entry=0x0, log_used=log_used@entry=0x0, log_ref=log_ref@entry=0, disable_memtable=disable_memtable@entry=false, seq_used=seq_used@entry=0x0, batch_cnt=batch_cnt@entry=0, pre_release_callback=pre_release_callback@entry=0x0) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:326
#8 0x000056375a95bf32 in rocksdb::DBImpl::Write (this=<optimized out>, write_options=..., my_batch=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/librocksdb_sys/rocksdb/db/db_impl/db_impl_write.cc:58
#9 0x000056375a87ddea in crocksdb_write (db=<optimized out>, options=<optimized out>, batch=<optimized out>, errptr=0x7fea031b9da0) at crocksdb/c.cc:1040
#10 0x00005637594b9595 in raftstore::store::fsm::apply::ApplyContext$LT$EK$C$W$GT$::write_to_db::h0da768c7781471b0 (self=<optimized out>) at /rust/git/checkouts/rust-rocksdb-a9a28e74c6ead8ef/c6b58a4/src/rocksdb.rs:790
#11 0x00005637594b8341 in raftstore::store::fsm::apply::ApplyContext$LT$EK$C$W$GT$::flush::hfbf6667b7288e6f4 (self=0x7fea031ba220) at components/raftstore/src/store/fsm/apply.rs:553
#12 0x00005637594b60c5 in std::sys_common::backtrace::__rust_begin_short_backtrace::h23596ebe92d23460 (f=...) at components/raftstore/src/store/fsm/apply.rs:3540
#13 0x00005637594b581c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hae5a836fa8f5db22 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#14 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#15 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#16 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#17 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#18 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 29 (Thread 0x7fea02fde700 (LWP 19021)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11ebc8 in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park_timeout () at library/std/src/sys_common/thread_parker/futex.rs:69
#3 park_timeout () at library/std/src/thread/mod.rs:945
#4 std::sync::mpsc::blocking::WaitToken::wait_max_until::h105066c0ee2201f2 () at library/std/src/sync/mpsc/blocking.rs:75
#5 0x000056375937ddae in std::sync::mpsc::oneshot::Packet$LT$T$GT$::recv::hc3360613b8d0d5d2 (self=0x7fea4ecabdc0, deadline=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/oneshot.rs:134
#6 0x000056375937cf59 in std::sync::mpsc::Receiver$LT$T$GT$::recv_timeout::h3bf2a9713205aa8e (self=0x7fea02fb9708, timeout=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/mod.rs:1306
#7 0x000056375951b030 in std::sys_common::backtrace::__rust_begin_short_backtrace::h50d885ba90e1127d (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/raftstore/src/store/worker/pd.rs:360
#8 0x000056375951ae6c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h1ff6e3652b7b242a () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#9 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#10 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#11 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#12 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#13 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 28 (Thread 0x7fea02ddd700 (LWP 19022)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4307bf50) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x00005637593cc670 in _$LT$tokio..park..either..Either$LT$A$C$B$GT$$u20$as$u20$tokio..park..Park$GT$::park::h013c3981d499a7f7 (self=0x7fea02db82b0) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x00005637598875c2 in std::sys_common::backtrace::__rust_begin_short_backtrace::h1cec2a7146b9b8e6 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/basic_scheduler.rs:158
#4 0x000056375988663c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h665a02b92888d3bc () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#5 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#8 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 27 (Thread 0x7fea02bdc700 (LWP 19023)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11e99a in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park () at library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at library/std/src/thread/mod.rs:881
#4 std::sync::mpsc::blocking::WaitToken::wait::hae3a13d09b5b5f41 () at library/std/src/sync/mpsc/blocking.rs:64
#5 0x0000563759495f1d in tikv::server::gc_worker::gc_manager::GcManager$LT$S$C$R$GT$::run::hd21346363a9450b9 (self=0x7fea02bb78d8) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/oneshot.rs:140
#6 0x000056375949390a in std::sys_common::backtrace::__rust_begin_short_backtrace::hca28b20f2a3272d3 (f=...) at src/server/gc_worker/gc_manager.rs:278
#7 0x00005637594937ac in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h3e68b939046577d8 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#8 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#10 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#11 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 26 (Thread 0x7fea029db700 (LWP 19024)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4f2b44d0) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a7549db in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea029b6810, core=0x7fea4f2b4580, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 25 (Thread 0x7fea027da700 (LWP 19025)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea027b5810, core=0x7fea4f2b4600, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 24 (Thread 0x7fea025d9700 (LWP 19026)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea025b4810, core=0x7fea4f2b4680, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x7fea023d8700 (LWP 19027)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea023b3810, core=0x7fea4f2b4740, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x7fea021d7700 (LWP 19028)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4ed54150) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a7549db in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea021b2810, core=0x7fea4ed54200, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x7fea01fd6700 (LWP 19029)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x7fea01dd5700 (LWP 19030)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11e99a in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park () at library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at library/std/src/thread/mod.rs:881
#4 std::sync::mpsc::blocking::WaitToken::wait::hae3a13d09b5b5f41 () at library/std/src/sync/mpsc/blocking.rs:64
#5 0x0000563759bdf910 in recv<futures_executor::thread_pool::Message> (self=0x7fea4927ce80, deadline=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/stream.rs:200
#6 recv<futures_executor::thread_pool::Message> (self=0x7fea0796c3a0) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/mod.rs:1128
#7 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#8 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#9 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#10 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#11 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#12 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#13 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#14 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#15 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#16 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#17 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#19 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#20 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#21 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x7fea01bd4700 (LWP 19031)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7fea019d3700 (LWP 19032)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7fea017d2700 (LWP 19033)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7fea015d1700 (LWP 19034)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7fea013d0700 (LWP 19035)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7fea011cf700 (LWP 19036)):
#0 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fea504dbe9b in _L_lock_883 () from /lib64/libpthread.so.0
#2 0x00007fea504dbd68 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000563759bdebc1 in lock (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/mutex.rs:65
#4 raw_lock (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/mutex.rs:76
#5 lock<std::sync::mpsc::Receiver<futures_executor::thread_pool::Message>> (self=0x7fea0796c390) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mutex.rs:263
#6 work (self=0x7fea0796c370, idx=<optimized out>, after_start=..., before_stop=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:158
#7 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.7/src/thread_pool.rs:288
#8 std::sys_common::backtrace::__rust_begin_short_backtrace::h4066e643cc782408 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#9 0x0000563759bde959 in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#10 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#11 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#12 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#13 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#14 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#15 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h15568a38ebe0af33 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#16 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#17 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#18 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#19 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#20 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7fea00fce700 (LWP 19037)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea4381e050) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a617910 in _$LT$tokio..park..either..Either$LT$A$C$B$GT$$u20$as$u20$tokio..park..Park$GT$::park::hcdf9c710aeb294ee (self=0x7fea00fa9800) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a612eb0 in std::sys_common::backtrace::__rust_begin_short_backtrace::h4208385c710bdfd4 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/basic_scheduler.rs:158
#4 0x000056375a611edd in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h6d45e87aefda0a12 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#5 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#6 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#8 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7fea00dcd700 (LWP 19038)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375ae3ad10 in pollable_epoll (deadline=<optimized out>, p=0x7fea4f2bfc00) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:949
#2 pollset_work (pollset=0x7fea4309c0c0, worker_hdl=<optimized out>, deadline=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/ev_epollex_linux.cc:1138
#3 0x000056375ad368e3 in cq_next (cq=0x7fea4309c000, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1060
#4 0x000056375ad36f2b in grpc_completion_queue_next (cq=<optimized out>, deadline=..., reserved=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/surface/completion_queue.cc:1135
#5 0x0000563759bec5de in std::sys_common::backtrace::__rust_begin_short_backtrace::haa843a8c70fce8b5 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-0.6.0/src/cq.rs:196
#6 0x0000563759bebdb5 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h2133dd76c7d83def () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7fea00bcc700 (LWP 19039)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11e99a in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park () at library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at library/std/src/thread/mod.rs:881
#4 std::sync::mpsc::blocking::WaitToken::wait::hae3a13d09b5b5f41 () at library/std/src/sync/mpsc/blocking.rs:64
#5 0x00005637593d0bf9 in tikv::server::lock_manager::deadlock::Detector$LT$S$C$P$GT$::handle_detect::haecc713652b380db (self=0x7fea00ba7858, tp=<optimized out>, txn_ts=..., lock=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/oneshot.rs:140
#6 0x00005637593caf13 in _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h0e9bac6a8e66d126 (self=..., cx=0x7fea00ba7370) at src/server/lock_manager/deadlock.rs:828
#7 0x00005637593c9d4a in std::sys_common::backtrace::__rust_begin_short_backtrace::h9a407ad01d06bc12 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/basic_scheduler.rs:131
#8 0x00005637593c900c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h5e7922109af5243b () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#9 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#10 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#11 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#12 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#13 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7fea009cb700 (LWP 19040)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x000056375a11ebc8 in futex_wait () at library/std/src/sys/unix/futex.rs:18
#2 park_timeout () at library/std/src/sys_common/thread_parker/futex.rs:69
#3 park_timeout () at library/std/src/thread/mod.rs:945
#4 std::sync::mpsc::blocking::WaitToken::wait_max_until::h105066c0ee2201f2 () at library/std/src/sync/mpsc/blocking.rs:75
#5 0x000056375937ddae in std::sync::mpsc::oneshot::Packet$LT$T$GT$::recv::hc3360613b8d0d5d2 (self=0x7fea0797b090, deadline=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/oneshot.rs:134
#6 0x000056375937cf59 in std::sync::mpsc::Receiver$LT$T$GT$::recv_timeout::h3bf2a9713205aa8e (self=0x7fea009a6990, timeout=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sync/mpsc/mod.rs:1306
#7 0x00005637597f29d1 in std::sys_common::backtrace::__rust_begin_short_backtrace::h45a0cff1583d1c66 (f=...) at /home/jenkins/agent/workspace/build_tikv_multi_branch_master/tikv/components/engine_traits/src/metrics_flusher.rs:47
#8 0x00005637597f2823 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hfd09545908b914fe () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#9 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#10 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#11 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#12 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#13 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7fea007ca700 (LWP 19041)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea007a5810, core=0x7fea079799c0, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7fea005c9700 (LWP 19042)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea005a4810, core=0x7fea07979a40, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7fea003c8700 (LWP 19043)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a7461c6 in tokio::park::thread::Inner::park::hdf90c68384ca305f (self=0x7fea07979910) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a7549db in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/park/thread.rs:60
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea003a3810, core=0x7fea07979ac0, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7fea001c7700 (LWP 19044)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375a754a63 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys/unix/condvar.rs:71
#2 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fea001a2810, core=0x7fea07979b40, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#3 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#4 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#5 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7fe9fffc6700 (LWP 19045)):
#0 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
#1 0x000056375a755561 in tokio::io::driver::Driver::turn::h8070e1a1ac13e1b8 (self=0x7fea0798f5c8, max_wait=...) at /rust/registry/src/github.com-1ecc6299db9ec823/mio-0.6.21/src/sys/unix/epoll.rs:72
#2 0x000056375a754c98 in _$LT$tokio..runtime..park..Parker$u20$as$u20$tokio..park..Park$GT$::park::h2de035f93c520f5b (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/io/driver/mod.rs:176
#3 0x000056375a753b5c in tokio::runtime::thread_pool::worker::Context::park_timeout::h2fef2ffb52041515 (self=0x7fe9fffa1810, core=0x7fea03bc03c0, duration=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:426
#4 0x000056375a751644 in tokio::runtime::task::raw::poll::hc2d52f3b93788c1b (ptr=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/thread_pool/worker.rs:402
#5 0x000056375a7507ca in std::sys_common::backtrace::__rust_begin_short_backtrace::h639a78f3b5965b39 (f=...) at /rust/registry/src/github.com-1ecc6299db9ec823/tokio-0.2.22/src/runtime/task/raw.rs:66
#6 0x000056375a7503d9 in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hb7edcac20ef92e3c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#7 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#9 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#10 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7fea491ff700 (LWP 19193)):
#0 0x00007fea504e0e9d in nanosleep () from /lib64/libpthread.so.0
#1 0x0000563759db5dd0 in prometheus::timer::ensure_updater::_$u7b$$u7b$closure$u7d$$u7d$::hfadc0ceb39a9060c () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5//library/std/src/sys/unix/thread.rs:181
#2 0x0000563759db5d66 in std::sys_common::backtrace::__rust_begin_short_backtrace::ha46b8a98378f02e6 (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#3 0x0000563759db5d56 in std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h25c20825017fad67 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#4 0x0000563759db5d46 in _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c210983c998e989 (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#5 0x0000563759db5c5c in core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hc2e671aa6b348653 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#6 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#7 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#8 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#9 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7fe9fe4c5700 (LWP 19194)):
#0 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
#1 0x0000563759be2c39 in futex_wait () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5//library/std/src/sys/unix/futex.rs:18
#2 park () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5//library/std/src/sys_common/thread_parker/futex.rs:50
#3 park () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5//library/std/src/thread/mod.rs:881
#4 run (timer=..., done=...) at /rust/registry/src/github.com-1ecc6299db9ec823/futures-timer-3.0.2/src/native/global.rs:78
#5 {{closure}} () at /rust/registry/src/github.com-1ecc6299db9ec823/futures-timer-3.0.2/src/native/global.rs:28
#6 std::sys_common::backtrace::__rust_begin_short_backtrace::h648bd9ecf8171f6d (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#7 0x0000563759be208c in {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:470
#8 call_once<(),closure-0> (self=..., _args=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:308
#9 do_call<std::panic::AssertUnwindSafe<closure-0>,()> (data=<optimized out>) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:381
#10 try<(),std::panic::AssertUnwindSafe<closure-0>> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panicking.rs:345
#11 catch_unwind<std::panic::AssertUnwindSafe<closure-0>,()> (f=...) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/panic.rs:382
#12 {{closure}}<closure-0,()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/thread/mod.rs:469
#13 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h349ebe32995fc750 () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#14 0x000056375a12188a in call_once<(),FnOnce<()>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#15 call_once<(),alloc::boxed::Box<FnOnce<()>>> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/alloc/src/boxed.rs:1042
#16 std::sys::unix::thread::Thread::new::thread_start::hb4b79d379b730058 () at library/std/src/sys/unix/thread.rs:89
#17 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#18 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7fe99b5ff700 (LWP 19223)):
#0 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000056375ad20292 in gpr_cv_wait (cv=0x7fea4f2bf0e0, mu=0x7fea4f2bf0a8, abs_deadline=...) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gpr/sync_posix.cc:130
#2 0x000056375ade7b8f in grpc_core::Executor::ThreadMain (arg=0x7fea4f2bf0a8) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/iomgr/executor.cc:231
#3 0x000056375ad4c98c in operator() (__closure=0x0, v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:140
#4 grpc_core::(anonymous namespace)::ThreadInternalsPosix::__lambda0::_FUN (v=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/grpcio-sys-0.6.0/grpc/src/core/lib/gprpp/thd_posix.cc:145
#5 0x000056375ad4c98c in grpc_core::Thread::Thread (this=<optimized out>, thd_name=<optimized out>, thd_body=<optimized out>, arg=<optimized out>, success=<optimized out>, options=...)
#6 0x00007fea504d9ea5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fea4f8de8dd in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7fea50901b80 (LWP 18882)):
#0 0x00007fea504e13c1 in sigwait () from /lib64/libpthread.so.0
#1 0x000056375a0d8ae9 in _$LT$signal..trap..Trap$u20$as$u20$core..iter..traits..iterator..Iterator$GT$::next::h1bb1b1419af7a3a9 (self=<optimized out>) at /rust/registry/src/github.com-1ecc6299db9ec823/signal-0.6.0/src/trap.rs:113
#2 0x00005637591cae69 in cmd::server::run_tikv::h78a777c45eedd08d (config=<error reading variable: Cannot access memory at address 0x0>) at cmd/src/signal_handler.rs:17
#3 0x00005637590a867e in tikv_server::main::h63d760f20d1a1f95 () at cmd/src/bin/tikv-server.rs:181
#4 0x000056375908fee3 in call_once<fn(),()> () at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/core/src/ops/function.rs:227
#5 std::sys_common::backtrace::__rust_begin_short_backtrace::h6bf973c710b92ecf (f=0x7fff7bff1b98) at /rustc/b1496c6e606dd908dd651ac2cce89815e10d7fc5/library/std/src/sys_common/backtrace.rs:125
#6 0x00005637590a9c65 in main ()
[Inferior 1 (process 18882) detached]
[New LWP 19223]
[New LWP 19194]
[New LWP 19193]
[New LWP 19045]
[New LWP 19044]
[New LWP 19043]
[New LWP 19042]
[New LWP 19041]
[New LWP 19040]
[New LWP 19039]
[New LWP 19038]
[New LWP 19037]
[New LWP 19036]
[New LWP 19035]
[New LWP 19034]
[New LWP 19033]
[New LWP 19032]
[New LWP 19031]
[New LWP 19030]
[New LWP 19029]
[New LWP 19028]
[New LWP 19027]
[New LWP 19026]
[New LWP 19025]
[New LWP 19024]
[New LWP 19023]
[New LWP 19022]
[New LWP 19021]
[New LWP 19020]
[New LWP 19019]
[New LWP 19018]
[New LWP 19017]
[New LWP 19016]
[New LWP 19015]
[New LWP 19014]
[New LWP 19013]
[New LWP 19012]
[New LWP 19011]
[New LWP 19010]
[New LWP 19009]
[New LWP 19008]
[New LWP 19007]
[New LWP 19006]
[New LWP 19005]
[New LWP 19004]
[New LWP 19003]
[New LWP 19002]
[New LWP 19001]
[New LWP 19000]
[New LWP 18999]
[New LWP 18998]
[New LWP 18997]
[New LWP 18996]
[New LWP 18995]
[New LWP 18979]
[New LWP 18978]
[New LWP 18977]
[New LWP 18976]
[New LWP 18975]
[New LWP 18957]
[New LWP 18956]
[New LWP 18955]
[New LWP 18954]
[New LWP 18950]
[New LWP 18943]
[New LWP 18916]
[New LWP 18915]
[New LWP 18914]
[New LWP 18913]
[New LWP 18912]
[New LWP 18911]
[New LWP 18910]
[New LWP 18909]
[New LWP 18908]
[New LWP 18907]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
0x00007fea504e13c1 in sigwait () from /lib64/libpthread.so.0
Id Target Id Frame
76 Thread 0x7fea4eb7f700 (LWP 18907) "slogger" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
75 Thread 0x7fea4e3ff700 (LWP 18908) "time-monitor" 0x00007fea504e0e9d in nanosleep () from /lib64/libpthread.so.0
74 Thread 0x7fea4ddff700 (LWP 18909) "default-executo" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
73 Thread 0x7fea4d1fe700 (LWP 18910) "resolver-execut" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
72 Thread 0x7fea4c7fd700 (LWP 18911) "grpc_global_tim" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
71 Thread 0x7fea4bdfc700 (LWP 18912) "grpc-server-0" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
70 Thread 0x7fea4bbfb700 (LWP 18913) "grpc-server-1" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
69 Thread 0x7fea4b3ff700 (LWP 18914) "grpc-server-2" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
68 Thread 0x7fea4b1fe700 (LWP 18915) "grpc-server-3" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
67 Thread 0x7fea4a9ff700 (LWP 18916) "grpc-server-4" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
66 Thread 0x7fea499ff700 (LWP 18943) "timer" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
65 Thread 0x7fea489ff700 (LWP 18950) "background-0" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
64 Thread 0x7fea475ff700 (LWP 18954) "rocksdb:low0" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
63 Thread 0x7fea467ff700 (LWP 18955) "rocksdb:low1" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
62 Thread 0x7fea45bfe700 (LWP 18956) "rocksdb:high0" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
61 Thread 0x7fea44bff700 (LWP 18957) "tikv-server" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
60 Thread 0x7fea2fff4700 (LWP 18975) "rocksdb:dump_st" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
59 Thread 0x7fea309f5700 (LWP 18976) "rocksdb:pst_st" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
58 Thread 0x7fea313f6700 (LWP 18977) "rocksdb:high1" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
57 Thread 0x7fea42fff700 (LWP 18978) "tikv-server" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
56 Thread 0x7fea41fff700 (LWP 18979) "grpc_global_tim" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
55 Thread 0x7fea2c3ee700 (LWP 18995) "rocksdb:dump_st" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
54 Thread 0x7fea2cdef700 (LWP 18996) "rocksdb:pst_st" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
53 Thread 0x7fea425fe700 (LWP 18997) "gc-worker" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
52 Thread 0x7fea369ff700 (LWP 18998) "lock-collector-" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
51 Thread 0x7fea367fe700 (LWP 18999) "cdc-0" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
50 Thread 0x7fea2d7f0700 (LWP 19000) "unified-read-po" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
49 Thread 0x7fea365fd700 (LWP 19001) "unified-read-po" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
48 Thread 0x7fea35bfc700 (LWP 19002) "unified-read-po" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
47 Thread 0x7fea351fb700 (LWP 19003) "unified-read-po" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
46 Thread 0x7fea347fa700 (LWP 19004) "debugger" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
45 Thread 0x7fea315f9700 (LWP 19005) "sched-worker-po" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
44 Thread 0x7fea065ed700 (LWP 19006) "sched-worker-po" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
43 Thread 0x7fea063ec700 (LWP 19007) "sched-worker-po" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
42 Thread 0x7fea061eb700 (LWP 19008) "sched-worker-po" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
41 Thread 0x7fea05fea700 (LWP 19009) "sched-high-pri-" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
40 Thread 0x7fea05de9700 (LWP 19010) "sched-high-pri-" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
39 Thread 0x7fea05be8700 (LWP 19011) "transport-stats" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
38 Thread 0x7fea059e7700 (LWP 19012) "snap-handler-0" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
37 Thread 0x7fea057e6700 (LWP 19013) "raft-stream-0" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
36 Thread 0x7fea055e5700 (LWP 19014) "snap-generator-" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
35 Thread 0x7fea053e4700 (LWP 19015) "snap-generator-" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
34 Thread 0x7fea051e3700 (LWP 19016) "steady-timer" 0x00007fea504ddde2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
33 Thread 0x7fea04fe2700 (LWP 19017) "raftstore-7-0" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
32 Thread 0x7fea035e1700 (LWP 19018) "raftstore-7-1" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
31 Thread 0x7fea033e0700 (LWP 19019) "apply-0" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
30 Thread 0x7fea031df700 (LWP 19020) "apply-1" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
29 Thread 0x7fea02fde700 (LWP 19021) "stats-monitor" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
28 Thread 0x7fea02ddd700 (LWP 19022) "pd-worker" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
27 Thread 0x7fea02bdc700 (LWP 19023) "gc-manager" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
26 Thread 0x7fea029db700 (LWP 19024) "cdcwkr" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
25 Thread 0x7fea027da700 (LWP 19025) "cdcwkr" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
24 Thread 0x7fea025d9700 (LWP 19026) "cdcwkr" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
23 Thread 0x7fea023d8700 (LWP 19027) "cdcwkr" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
22 Thread 0x7fea021d7700 (LWP 19028) "tso" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
21 Thread 0x7fea01fd6700 (LWP 19029) "sst-importer0" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
20 Thread 0x7fea01dd5700 (LWP 19030) "sst-importer1" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
19 Thread 0x7fea01bd4700 (LWP 19031) "sst-importer2" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
18 Thread 0x7fea019d3700 (LWP 19032) "sst-importer3" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
17 Thread 0x7fea017d2700 (LWP 19033) "sst-importer4" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
16 Thread 0x7fea015d1700 (LWP 19034) "sst-importer5" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
15 Thread 0x7fea013d0700 (LWP 19035) "sst-importer6" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
14 Thread 0x7fea011cf700 (LWP 19036) "sst-importer7" 0x00007fea504e054d in __lll_lock_wait () from /lib64/libpthread.so.0
13 Thread 0x7fea00fce700 (LWP 19037) "waiter-manager" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
12 Thread 0x7fea00dcd700 (LWP 19038) "deadlock-0" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
11 Thread 0x7fea00bcc700 (LWP 19039) "deadlock-detect" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
10 Thread 0x7fea009cb700 (LWP 19040) "metrics-flusher" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
9 Thread 0x7fea007ca700 (LWP 19041) "snap-sender" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
8 Thread 0x7fea005c9700 (LWP 19042) "snap-sender" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
7 Thread 0x7fea003c8700 (LWP 19043) "snap-sender" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
6 Thread 0x7fea001c7700 (LWP 19044) "snap-sender" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
5 Thread 0x7fe9fffc6700 (LWP 19045) "status-server" 0x00007fea4f8deeb3 in epoll_wait () from /lib64/libc.so.6
4 Thread 0x7fea491ff700 (LWP 19193) "time updater" 0x00007fea504e0e9d in nanosleep () from /lib64/libpthread.so.0
3 Thread 0x7fe9fe4c5700 (LWP 19194) "futures-timer" 0x00007fea4f8d8bf9 in syscall () from /lib64/libc.so.6
2 Thread 0x7fe99b5ff700 (LWP 19223) "default-executo" 0x00007fea504dda35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
* 1 Thread 0x7fea50901b80 (LWP 18882) "tikv-server" 0x00007fea504e13c1 in sigwait () from /lib64/libpthread.so.0
[Inferior 1 (process 18882) detached]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment