Created
January 15, 2016 09:13
-
-
Save daverigby/fb9dc02d04fb02954231 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Related to: http://review.couchbase.org/#/c/58557/ | |
| From: http://factory.couchbase.com/job/ep-engine-threadsanitizer-master/1268/consoleText | |
| WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=119209) | |
| Cycle in lock order graph: M19289 (0x7d540000f8f0) => M19325 (0x7d480000b8e0) => M15588 (0x7d6400007fa8) => M19289 | |
| Mutex M19325 acquired here while holding mutex M19289 in main thread: | |
| #0 pthread_mutex_lock <null> (engine_testapp+0x0000004865a0) | |
| #1 std::mutex::lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/4.9/../../../../include/x86_64-linux-gnu/c++/4.9/bits/gthr-default.h:748 (ep.so+0x000000062fa2) | |
| #2 EvpDcpStreamReq(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:1551 (ep.so+0x0000000b3707) | |
| #3 mock_dcp_stream_req(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:551 (engine_testapp+0x0000004ce9e7) | |
| #4 dcp_stream(engine_interface*, engine_interface_v1*, char const*, void const*, unsigned short, unsigned int, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, int, int, int, int, bool, bool, unsigned char, bool, unsigned long*, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:2339 (ep_testsuite.so+0x000000081396) | |
| #5 test_dcp_takeover(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:3308 (ep_testsuite.so+0x000000065daa) | |
| #6 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:1134 (engine_testapp+0x0000004cc728) | |
| #7 __libc_start_main /build/buildd/eglibc-2.15/csu/libc-start.c:226 (libc.so.6+0x00000002176c) | |
| Mutex M19289 previously acquired by the same thread here: | |
| #0 pthread_rwlock_wrlock <null> (engine_testapp+0x00000046a310) | |
| #1 cb_rw_writer_enter /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:415 (libplatform.so.0.1.0+0x00000000695d) | |
| #2 WriterLockHolder::WriterLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/rwlock.h:47 (ep.so+0x000000067993) | |
| #3 DcpProducer::streamRequest(unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/producer.cc:295 (ep.so+0x000000062ef7) | |
| #4 EvpDcpStreamReq(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:1551 (ep.so+0x0000000b3707) | |
| #5 mock_dcp_stream_req(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:551 (engine_testapp+0x0000004ce9e7) | |
| #6 dcp_stream(engine_interface*, engine_interface_v1*, char const*, void const*, unsigned short, unsigned int, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, int, int, int, int, bool, bool, unsigned char, bool, unsigned long*, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:2339 (ep_testsuite.so+0x000000081396) | |
| #7 test_dcp_takeover(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:3308 (ep_testsuite.so+0x000000065daa) | |
| #8 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:1134 (engine_testapp+0x0000004cc728) | |
| #9 __libc_start_main /build/buildd/eglibc-2.15/csu/libc-start.c:226 (libc.so.6+0x00000002176c) | |
| Mutex M15588 acquired here while holding mutex M19325 in thread T16: | |
| #0 pthread_rwlock_rdlock <null> (engine_testapp+0x000000469e20) | |
| #1 cb_rw_reader_enter /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:391 (libplatform.so.0.1.0+0x00000000687d) | |
| #2 ReaderLockHolder::ReaderLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/rwlock.h:39 (ep.so+0x000000067c33) | |
| #3 ActiveStream::isCurrentSnapshotCompleted() const /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:1058 (ep.so+0x0000000733f6) | |
| #4 ActiveStream::snapshot(std::deque<MutationResponse*, std::allocator<MutationResponse*> >&, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:756 (ep.so+0x000000072ceb) | |
| #5 ActiveStream::nextCheckpointItemTask() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:743 (ep.so+0x0000000726f0) | |
| #6 ActiveStreamCheckpointProcessorTask::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:653 (ep.so+0x000000071f28) | |
| #7 ExecutorThread::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorthread.cc:115 (ep.so+0x0000000f1166) | |
| #8 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorthread.cc:33 (ep.so+0x0000000f0d15) | |
| #9 platform_thread_wrap(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:54 (libplatform.so.0.1.0+0x00000000551b) | |
| Mutex M19325 previously acquired by the same thread here: | |
| #0 pthread_mutex_lock <null> (engine_testapp+0x0000004865a0) | |
| #1 std::mutex::lock() /usr/bin/../lib/gcc/x86_64-linux-gnu/4.9/../../../../include/x86_64-linux-gnu/c++/4.9/bits/gthr-default.h:748 (ep.so+0x000000072cd6) | |
| #2 ActiveStream::nextCheckpointItemTask() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:743 (ep.so+0x0000000726f0) | |
| #3 ActiveStreamCheckpointProcessorTask::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/stream.cc:653 (ep.so+0x000000071f28) | |
| #4 ExecutorThread::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorthread.cc:115 (ep.so+0x0000000f1166) | |
| #5 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorthread.cc:33 (ep.so+0x0000000f0d15) | |
| #6 platform_thread_wrap(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:54 (libplatform.so.0.1.0+0x00000000551b) | |
| Mutex M19289 acquired here while holding mutex M15588 in main thread: | |
| #0 pthread_rwlock_rdlock <null> (engine_testapp+0x000000469e20) | |
| #1 cb_rw_reader_enter /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:391 (libplatform.so.0.1.0+0x00000000687d) | |
| #2 ReaderLockHolder::ReaderLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/rwlock.h:39 (ep.so+0x000000067c33) | |
| #3 DcpProducer::notifySeqnoAvailable(unsigned short, unsigned long) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/dcp/producer.cc:987 (ep.so+0x00000006679d) | |
| #4 DcpConnMap::notifyVBConnections(unsigned short, unsigned long) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/connmap.cc:1196 (ep.so+0x000000047116) | |
| #5 EventuallyPersistentStore::queueDirty(RCPtr<VBucket>&, StoredValue*, LockHolder*, unsigned long*, bool, bool, bool, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep.cc:3551 (ep.so+0x0000000853eb) | |
| #6 EventuallyPersistentStore::set(Item const&, void const*, bool, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep.cc:866 (ep.so+0x000000086ecc) | |
| #7 EventuallyPersistentEngine::store(void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:2268 (ep.so+0x0000000b5b2f) | |
| #8 EvpStore(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:235 (ep.so+0x0000000b2877) | |
| #9 mock_store(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:274 (engine_testapp+0x0000004cdce2) | |
| #10 storeCasVb11(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_test_apis.cc:853 (ep_testsuite.so+0x00000009a3af) | |
| #11 store(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_test_apis.cc:804 (ep_testsuite.so+0x0000000984cb) | |
| #12 dcp_stream(engine_interface*, engine_interface_v1*, char const*, void const*, unsigned short, unsigned int, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, int, int, int, int, bool, bool, unsigned char, bool, unsigned long*, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:2523 (ep_testsuite.so+0x000000082723) | |
| #13 test_dcp_takeover(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:3308 (ep_testsuite.so+0x000000065daa) | |
| #14 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:1134 (engine_testapp+0x0000004cc728) | |
| #15 __libc_start_main /build/buildd/eglibc-2.15/csu/libc-start.c:226 (libc.so.6+0x00000002176c) | |
| Mutex M15588 previously acquired by the same thread here: | |
| #0 pthread_rwlock_rdlock <null> (engine_testapp+0x000000469e20) | |
| #1 cb_rw_reader_enter /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:391 (libplatform.so.0.1.0+0x00000000687d) | |
| #2 ReaderLockHolder::ReaderLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/rwlock.h:39 (ep.so+0x000000067c33) | |
| #3 EventuallyPersistentStore::set(Item const&, void const*, bool, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep.cc:798 (ep.so+0x0000000868d7) | |
| #4 EventuallyPersistentEngine::store(void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:2268 (ep.so+0x0000000b5b2f) | |
| #5 EvpStore(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:235 (ep.so+0x0000000b2877) | |
| #6 mock_store(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:274 (engine_testapp+0x0000004cdce2) | |
| #7 storeCasVb11(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_test_apis.cc:853 (ep_testsuite.so+0x00000009a3af) | |
| #8 store(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_test_apis.cc:804 (ep_testsuite.so+0x0000000984cb) | |
| #9 dcp_stream(engine_interface*, engine_interface_v1*, char const*, void const*, unsigned short, unsigned int, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, int, int, int, int, bool, bool, unsigned char, bool, unsigned long*, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:2523 (ep_testsuite.so+0x000000082723) | |
| #10 test_dcp_takeover(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/tests/ep_testsuite.cc:3308 (ep_testsuite.so+0x000000065daa) | |
| #11 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:1134 (engine_testapp+0x0000004cc728) | |
| #12 __libc_start_main /build/buildd/eglibc-2.15/csu/libc-start.c:226 (libc.so.6+0x00000002176c) | |
| Thread T16 'mc:auxio_15' (tid=119227, running) created by main thread at: | |
| #0 pthread_create <null> (engine_testapp+0x000000467751) | |
| #1 cb_create_named_thread /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/platform/src/cb_pthreads.cc:104 (libplatform.so.0.1.0+0x000000005375) | |
| #2 ExecutorThread::start() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorthread.cc:46 (ep.so+0x0000000f06b7) | |
| #3 ExecutorPool::_startWorkers() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorpool.cc:522 (ep.so+0x0000000ea417) | |
| #4 ExecutorPool::_registerTaskable(Taskable&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorpool.cc:479 (ep.so+0x0000000e9689) | |
| #5 ExecutorPool::registerTaskable(Taskable&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/executorpool.cc:484 (ep.so+0x0000000eb8be) | |
| #6 EventuallyPersistentStore::EventuallyPersistentStore(EventuallyPersistentEngine&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep.cc:301 (ep.so+0x00000007f6ba) | |
| #7 EventuallyPersistentEngine::initialize(char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:2163 (ep.so+0x0000000b4b33) | |
| #8 EvpInitialize(engine_interface*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/ep-engine/src/ep_engine.cc:139 (ep.so+0x0000000b2468) | |
| #9 init_engine_instance <null> (libmcd_util.so.1.0.0+0x00000000586b) | |
| #10 create_bucket(bool, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:982 (engine_testapp+0x0000004cd577) | |
| #11 execute_test(test, char const*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-master/memcached/programs/engine_testapp/engine_testapp.cc:1127 (engine_testapp+0x0000004cc690) | |
| #12 __libc_start_main /build/buildd/eglibc-2.15/csu/libc-start.c:226 (libc.so.6+0x00000002176c) | |
| SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) ??:0 pthread_mutex_lock |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment