Skip to content

Instantly share code, notes, and snippets.

@godmar
Created November 12, 2019 17:47
Show Gist options
  • Save godmar/a52562a04acc8a3e529fdf4da08ef9e6 to your computer and use it in GitHub Desktop.
Save godmar/a52562a04acc8a3e529fdf4da08ef9e6 to your computer and use it in GitHub Desktop.
The copied and pasted fragments below show what happens when wasm-ld locks up in eosio.cdt 1.6.1
Below is a backtrace of the threadpool-related deadlock that occurs in eosio.cdt 1.6.1 while linking with parallel linking enabled.
I've also included the relevant code sections in LLVM and the fix.
(gdb) attach 28855
Attaching to program: /usr/opt/eosio.cdt/1.6.1/bin/wasm-ld, process 28855
[New LWP 28856]
[New LWP 28857]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7ffe6af477d8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
88 ../sysdeps/unix/sysv/linux/futex-internal.h: No such file or directory.
(gdb) bt
#0 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7ffe6af477d8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1 __pthread_cond_wait_common (abstime=0x0, mutex=0x7ffe6af47788, cond=0x7ffe6af477b0) at pthread_cond_wait.c:502
#2 __pthread_cond_wait (cond=0x7ffe6af477b0, mutex=0x7ffe6af47788) at pthread_cond_wait.c:655
#3 0x00007f67ca4a286c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00005649734d3273 in llvm::parallel::detail::Latch::sync() const ()
#5 0x00005649736e6445 in lld::wasm::writeResult(bool) ()
#6 0x00005649736a350e in (anonymous namespace)::LinkerDriver::link(llvm::ArrayRef<char const*>) [clone .constprop.478] ()
#7 0x00005649736a3ba5 in lld::wasm::link(llvm::ArrayRef<char const*>, bool, llvm::raw_ostream&) ()
#8 0x0000564973416d58 in main ()
(gdb) info threads
Id Target Id Frame
* 1 Thread 0x7f67caff5b80 (LWP 28855) "wasm-ld" 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0,
futex_word=0x7ffe6af477d8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
2 Thread 0x7f67c9a43700 (LWP 28856) "wasm-ld" 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0,
futex_word=0x7f67c9a42da8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
3 Thread 0x7f67c9242700 (LWP 28857) "wasm-ld" 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0,
futex_word=0x7f67c9241db8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
(gdb) thread 2
[Switching to thread 2 (Thread 0x7f67c9a43700 (LWP 28856))]
#0 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f67c9a42da8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
88 in ../sysdeps/unix/sysv/linux/futex-internal.h
(gdb) bt
#0 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f67c9a42da8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1 __pthread_cond_wait_common (abstime=0x0, mutex=0x7f67c9a42d58, cond=0x7f67c9a42d80) at pthread_cond_wait.c:502
#2 __pthread_cond_wait (cond=0x7f67c9a42d80, mutex=0x7f67c9a42d58) at pthread_cond_wait.c:655
#3 0x00007f67ca4a286c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00005649736ee0db in lld::wasm::CustomSection::writeTo(unsigned char*) ()
#5 0x00005649736b7d08 in std::_Function_handler<void (), void llvm::parallel::detail::parallel_for_each<__gnu_cxx::__normal_iterator<lld::wasm::OutputSection**, std::vector<lld::wasm::OutputSection*, std::allocator<lld::wasm::OutputSection*> > >, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1}>(__gnu_cxx::__normal_iterator<lld::wasm::OutputSection**, std::vector<lld::wasm::OutputSection*, std::allocator<lld::wasm::OutputSection*> > >, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1}, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1})::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#6 0x0000564974b1e2c6 in std::_Function_handler<void (), llvm::parallel::detail::TaskGroup::spawn(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#7 0x0000564974b1ea35 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<(anonymous namespace)::ThreadPoolExecutor::ThreadPoolExecutor(unsigned int)::{lambda()#1}> > >::_M_run() ()
#8 0x00007f67ca4a866f in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#9 0x00007f67cabc26db in start_thread (arg=0x7f67c9a43700) at pthread_create.c:463
#10 0x00007f67c9b6588f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) thread 3
[Switching to thread 3 (Thread 0x7f67c9242700 (LWP 28857))]
#0 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f67c9241db8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
88 in ../sysdeps/unix/sysv/linux/futex-internal.h
(gdb) bt
#0 0x00007f67cabc89f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f67c9241db8)
at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1 __pthread_cond_wait_common (abstime=0x0, mutex=0x7f67c9241d68, cond=0x7f67c9241d90) at pthread_cond_wait.c:502
#2 __pthread_cond_wait (cond=0x7f67c9241d90, mutex=0x7f67c9241d68) at pthread_cond_wait.c:655
#3 0x00007f67ca4a286c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x00005649736ed90d in lld::wasm::DataSection::writeTo(unsigned char*) ()
#5 0x00005649736b7d08 in std::_Function_handler<void (), void llvm::parallel::detail::parallel_for_each<__gnu_cxx::__normal_iterator<lld::wasm::OutputSection**, std::vector<lld::wasm::OutputSection*, std::allocator<lld::wasm::OutputSection*> > >, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1}>(__gnu_cxx::__normal_iterator<lld::wasm::OutputSection**, std::vector<lld::wasm::OutputSection*, std::allocator<lld::wasm::OutputSection*> > >, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1}, (anonymous namespace)::Writer::writeSections()::{lambda(lld::wasm::OutputSection*)#1})::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#6 0x0000564974b1e2c6 in std::_Function_handler<void (), llvm::parallel::detail::TaskGroup::spawn(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
#7 0x0000564974b1e43d in std::thread::_State_impl<std::thread::_Invoker<std::tuple<(anonymous namespace)::ThreadPoolExecutor::ThreadPoolExecutor(unsigned int)::{lambda()#1}::operator()() const::{lambda()#1}> > >::_M_run() ()
#8 0x00007f67ca4a866f in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#9 0x00007f67cabc26db in start_thread (arg=0x7f67c9242700) at pthread_create.c:463
#10 0x00007f67c9b6588f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Threads 2 and 3 both try to run parallelForEach below, one for regular sections, one for custom section:
void CodeSection::writeTo(uint8_t *Buf) {
log("writing " + toString(*this));
log(" size=" + Twine(getSize()));
log(" headersize=" + Twine(Header.size()));
log(" codeheadersize=" + Twine(CodeSectionHeader.size()));
Buf += Offset;
// Write section header
memcpy(Buf, Header.data(), Header.size());
Buf += Header.size();
// Write code section headers
memcpy(Buf, CodeSectionHeader.data(), CodeSectionHeader.size());
// Write code section bodies
parallelForEach(Functions, <<<<<<<<<<< Thread 2 and 3
[&](const InputChunk *Chunk) { Chunk->writeTo(Buf); });
}
See eosio.cdt/eosio_llvm/tools/lld/wasm/Writer.cpp
void Writer::writeSections() {
uint8_t *Buf = Buffer->getBufferStart();
parallelForEach(OutputSections, [Buf](OutputSection *S) { S->writeTo(Buf); });
}
See https://github.com/llvm/llvm-project/commit/5081e41bdae2eb14a3f3eb8810263f9fea8fc7c1 for the fix
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment