error_message

gistfile1.txt

Traceback (most recent call last):
  File "topi/tests/python/test_topi_vision.py", line 437, in <module>
    test_get_valid_counts()
  File "topi/tests/python/test_topi_vision.py", line 84, in test_get_valid_counts
    verify_get_valid_counts((1, 15360, 5), 0)
  File "topi/tests/python/test_topi_vision.py", line 74, in verify_get_valid_counts
    check_device(device)
  File "topi/tests/python/test_topi_vision.py", line 70, in check_device
    tvm.testing.assert_allclose(tvm_out1.asnumpy(), np_out1, rtol=1e-3)
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/ndarray.py", line 283, in asnumpy
    check_call(_LIB.TVMArrayCopyToBytes(self.handle, data, nbytes))
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/base.py", line 314, in check_call
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (3) /home/ubuntu/workplace/tvm/build/libtvm.so(TVMArrayCopyToBytes+0x19a) [0x7f5e658e397a]
  [bt] (2) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDADeviceAPI::CopyDataFromTo(void const*, unsigned long, void*, unsigned long, unsigned long, DLContext, DLContext, DLDataType, void*)+0x29c) [0x7f5e659419a4]
  [bt] (1) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDADeviceAPI::GPUCopy(void const*, void*, unsigned long, cudaMemcpyKind, CUstream_st*)+0x297) [0x7f5e65942a00]
  [bt] (0) /home/ubuntu/workplace/tvm/build/libtvm.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x34) [0x7f5e64e1f57e]
  File "/home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc", line 215
CUDA: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading: an illegal memory access was encountered
[21:35:52] /home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc:214: 0x56547c829750,0x1204658000,307200,2
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/_ctypes/ndarray.py", line 82, in __del__
    check_call(_LIB.TVMArrayFree(self.handle))
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/base.py", line 314, in check_call
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (4) /home/ubuntu/workplace/tvm/build/libtvm.so(TVMArrayFree+0x19) [0x7f5e658e3355]
  [bt] (3) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Container::DecRef()+0x67) [0x7f5e64e1fef9]
  [bt] (2) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Internal::DefaultDeleter(tvm::runtime::NDArray::Container*)+0x6b) [0x7f5e658e4024]
  [bt] (1) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDADeviceAPI::FreeDataSpace(DLContext, void*)+0x18e) [0x7f5e659416ac]
  [bt] (0) /home/ubuntu/workplace/tvm/build/libtvm.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x34) [0x7f5e64e1f57e]
  File "/home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc", line 122
CUDA: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading: an illegal memory access was encountered
[21:35:52] /home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc:214: 0x56547bf39870,0x1203d20000,4,2
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/_ctypes/ndarray.py", line 82, in __del__
    check_call(_LIB.TVMArrayFree(self.handle))
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/base.py", line 314, in check_call
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (4) /home/ubuntu/workplace/tvm/build/libtvm.so(TVMArrayFree+0x19) [0x7f5e658e3355]
  [bt] (3) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Container::DecRef()+0x67) [0x7f5e64e1fef9]
  [bt] (2) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Internal::DefaultDeleter(tvm::runtime::NDArray::Container*)+0x6b) [0x7f5e658e4024]
  [bt] (1) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDADeviceAPI::FreeDataSpace(DLContext, void*)+0x18e) [0x7f5e659416ac]
  [bt] (0) /home/ubuntu/workplace/tvm/build/libtvm.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x34) [0x7f5e64e1f57e]
  File "/home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc", line 122
CUDA: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading: an illegal memory access was encountered
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/_ctypes/ndarray.py", line 82, in __del__
    check_call(_LIB.TVMArrayFree(self.handle))
  File "/home/ubuntu/workplace/tvm/python/tvm/_ffi/base.py", line 314, in check_call
    raise get_last_ffi_error()
tvm._ffi.base.TVMError: Traceback (most recent call last):
  [bt] (4) /home/ubuntu/workplace/tvm/build/libtvm.so(TVMArrayFree+0x19) [0x7f5e658e3355]
  [bt] (3) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Container::DecRef()+0x67) [0x7f5e64e1fef9]
  [bt] (2) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::NDArray::Internal::DefaultDeleter(tvm::runtime::NDArray::Container*)+0x6b) [0x7f5e658e4024]
  [bt] (1) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDADeviceAPI::FreeDataSpace(DLContext, void*)+0x18e) [0x7f5e659416ac]
  [bt] (0) /home/ubuntu/workplace/tvm/build/libtvm.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x34) [0x7f5e64e1f57e]
  File "/home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_device_api.cc", line 122
CUDA: Check failed: e == cudaSuccess || e == cudaErrorCudartUnloading: an illegal memory access was encountered
terminate called after throwing an instance of 'dmlc::Error'
  what():  [21:35:52] /home/ubuntu/workplace/tvm/src/runtime/cuda/cuda_module.cc:61: CUDAError: cuModuleUnload(module_[i]) failed with error: CUDA_ERROR_ILLEGAL_ADDRESS
Stack trace:
  [bt] (0) /home/ubuntu/workplace/tvm/build/libtvm.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x34) [0x7f5e64e1f57e]
  [bt] (1) /home/ubuntu/workplace/tvm/build/libtvm.so(tvm::runtime::CUDAModuleNode::~CUDAModuleNode()+0x1fd) [0x7f5e65944df9]
  [bt] (2) /home/ubuntu/workplace/tvm/build/libtvm.so(void __gnu_cxx::new_allocator<tvm::runtime::CUDAModuleNode>::destroy<tvm::runtime::CUDAModuleNode>(tvm::runtime::CUDAModuleNode*)+0x23) [0x7f5e6594dc23]
  [bt] (3) /home/ubuntu/workplace/tvm/build/libtvm.so(void std::allocator_traits<std::allocator<tvm::runtime::CUDAModuleNode> >::destroy<tvm::runtime::CUDAModuleNode>(std::allocator<tvm::runtime::CUDAModuleNode>&, tvm::runtime::CUDAModuleNode*)+0x23) [0x7f5e6594dbef]
  [bt] (4) /home/ubuntu/workplace/tvm/build/libtvm.so(std::_Sp_counted_ptr_inplace<tvm::runtime::CUDAModuleNode, std::allocator<tvm::runtime::CUDAModuleNode>, (__gnu_cxx::_Lock_policy)2>::_M_dispose()+0x37) [0x7f5e6594dad9]
  [bt] (5) /home/ubuntu/workplace/tvm/build/libtvm.so(std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release()+0x42) [0x7f5e64e24a52]
  [bt] (6) /home/ubuntu/workplace/tvm/build/libtvm.so(std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count()+0x27) [0x7f5e64e22f4f]
  [bt] (7) /home/ubuntu/workplace/tvm/build/libtvm.so(std::__shared_ptr<tvm::runtime::ModuleNode, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr()+0x1c) [0x7f5e64e1fe72]
  [bt] (8) /home/ubuntu/workplace/tvm/build/libtvm.so(std::shared_ptr<tvm::runtime::ModuleNode>::~shared_ptr()+0x18) [0x7f5e64e1fe8e]


Aborted (core dumped)