gistfile1.txt

#0  0x00007fffe310eb30 in clock_gettime ()
#1  0x00007f610869f936 in __GI___clock_gettime (clock_id=4, tp=0x7fffe30224d0) at ../sysdeps/unix/clock_gettime.c:115
#2  0x00007f609491081e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007f60949f4d14 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007f60948adef9 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#5  0x00007f60948ae0b0 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#6  0x00007f60948dd623 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007f6094a76790 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#8  0x00007f60947e03e7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#9  0x00007f60947e0850 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#10 0x00007f60947e093e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#11 0x00007f60949a0fc2 in cuLaunchKernel () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#12 0x00007f60d29c7547 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#13 0x00007f60d29c75d7 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#14 0x00007f60d29fe92b in cudaLaunchKernel () from /usr/local/cuda/lib64/libcudart.so.10.2
#15 0x00007f60b9cdc502 in ncclBarrierEnqueueWait (comm=0x7f6068000e00) at enqueue.cc:215
#16 0x00007f60b9cde35b in ncclGroupEnd () at group.cc:282
#17 0x00007f60cf3cd7c9 in c10d::(anonymous namespace)::AutoNcclGroup::~AutoNcclGroup() [clone .isra.413] () from /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#18 0x00007f60cf3e062d in ...
#19 0x00007f60cf3e0e38 in c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) ()
   from /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_python.so

gistfile2.txt

#0  0x00007ffd6ab5cb30 in clock_gettime ()
#1  0x00007fa4fbb2f936 in __GI___clock_gettime (clock_id=4, tp=0x7ffd6aa19ca0) at ../sysdeps/unix/clock_gettime.c:115
#2  0x00007fa487da081e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007fa487e84d14 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007fa487d3def9 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#5  0x00007fa487d7b7af in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#6  0x00007fa487cb0668 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007fa487cb0dcc in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#8  0x00007fa4c5e68be7 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#9  0x00007fa4c5e610a0 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#10 0x00007fa4c5e6bac2 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#11 0x00007fa4c5e71031 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#12 0x00007fa4c5e632de in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#13 0x00007fa4c5e4d925 in ?? () from /usr/local/cuda/lib64/libcudart.so.10.2
#14 0x00007fa4c5e887c5 in cudaStreamCreateWithPriority () from /usr/local/cuda/lib64/libcudart.so.10.2
#15 0x00007fa4a947e76e in c10::cuda::(anonymous namespace)::initDeviceStreamState(signed char) () from /opt/conda/lib/python3.6/site-packages/torch/lib/libc10_cuda.so
#16 0x00007fa4fbdf2a99 in __pthread_once_slow (once_control=0x7fa4a96ab560 <c10::cuda::(anonymous namespace)::device_flags>, init_routine=0x7fa4c5b5a7ba <std::__once_proxy()>)
    at pthread_once.c:116
#17 0x00007fa4a9479bb9 in c10::cuda::getStreamFromPool(bool, signed char) () from /opt/conda/lib/python3.6/site-packages/torch/lib/libc10_cuda.so
#18 0x00007fa4c28760f3 in c10d::ProcessGroupNCCL::FutureNCCL::addCallback(std::function<void ()>) () from /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#19 0x00007fa4c2870442 in ...
#20 0x00007fa4c2870e38 in c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) ()
   from /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_python.so