trace.log

terminate called after throwing an instance of 'std::runtime_error'
  what():  cudaDeviceSynchronize() error( cudaErrorIllegalAddress): an illegal memory access was encountered /path/to/Trilinos/packages/kokkos/core/src/Cuda/Kokkos_Cuda_Impl.cpp:122
Traceback functionality not available


Program received signal SIGABRT: Process abort signal.

Backtrace for this error:
#0  0x7F22C45C2467
#1  0x7F22C45C2AAE
#2  0x7F22C38B166F
#3  0x7F22C38B15F7
#4  0x7F22C38B2CE7
#5  0x7F22CDBC29D4
#6  0x7F22CDBC0945
#7  0x7F22CDBC0972
#8  0x7F22CDBC0B92
#9  0x1A4A233 in Kokkos::Impl::throw_runtime_exception(std::string const&) at Kokkos_Error.cpp:72
#10  0x1A511CF in Kokkos::Impl::cuda_internal_error_throw(cudaError, char const*, char const*, int) at Kokkos_Cuda_Impl.cpp:133 (discriminator 1)
#11  0x14783AA in Kokkos::Impl::ParallelReduce<Tpetra::Details::CrsMatrixGetDiagCopyFunctor<Kokkos::View<double*, Kokkos::LayoutLeft, Kokkos::Device<Kokkos::Cuda, Kokkos::CudaUVMSpace>, Kokkos::MemoryTraits<1u> >, Tpetra::Details::LocalMap<int, int, Kokkos::Device<Kokkos::Cuda, Kokkos::CudaUVMSpace> >, KokkosSparse::CrsMatrix<double, int, Kokkos::Cuda, void, unsigned int> >, Kokkos::RangePolicy<Kokkos::Cuda, int>, Kokkos::InvalidType, Kokkos::Cuda>::execute() at Kokkos_Cuda_Parallel.hpp:824
#12  0x1478B86 in parallel_reduce<Kokkos::RangePolicy<Kokkos::Cuda, int>, Tpetra::Details::CrsMatrixGetDiagCopyFunctor<Kokkos::View<double*, Kokkos::LayoutLeft, Kokkos::Device<Kokkos::Cuda, Kokkos::CudaUVMSpace>, Kokkos::MemoryTraits<1u> >, Tpetra::Details::LocalMap<int, int, Kokkos::Device<Kokkos::Cuda, Kokkos::CudaUVMSpace> >, KokkosSparse::CrsMatrix<double, int, Kokkos::Cuda, void, unsigned int> >, int> at Kokkos_Parallel_Reduce.hpp:1197
#13  0xE45353 in Ifpack2::Relaxation<Tpetra::RowMatrix<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> > >::compute() at Ifpack2_Relaxation_def.hpp:898
#14  0xCD18D9 in MueLu::Ifpack2Smoother<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::SetupGeneric(MueLu::Level&) at MueLu_Ifpack2Smoother_def.hpp:444
#15  0xCD2094 in MueLu::Ifpack2Smoother<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::Setup(MueLu::Level&) at MueLu_Ifpack2Smoother_def.hpp:143
#16  0xB29A03 in MueLu::TrilinosSmoother<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::Setup(MueLu::Level&) at MueLu_TrilinosSmoother_def.hpp:177
#17  0xAD01B9 in MueLu::SmootherFactory<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::BuildSmoother(MueLu::Level&, MueLu::PreOrPost) const at MueLu_SmootherFactory_def.hpp:163
#18  0x6845B4 in MueLu::SingleLevelFactoryBase::CallBuild(MueLu::Level&) const at MueLu_SingleLevelFactoryBase.hpp:135
#19  0x99E336 in Teuchos::RCP<MueLu::SmootherBase<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> > >& MueLu::Level::Get<Teuchos::RCP<MueLu::SmootherBase<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> > > >(std::string const&, MueLu::FactoryBase const*) at MueLu_Level.hpp:203
#20  0xB1DE88 in MueLu::TopSmootherFactory<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::Build(MueLu::Level&) const at MueLu_TopSmootherFactory_def.hpp:98 (discriminator 2)
#21  0x9B5D6C in MueLu::Hierarchy<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::Setup(int, Teuchos::RCP<MueLu::FactoryManagerBase const>, Teuchos::RCP<MueLu::FactoryManagerBase const>, Teuchos::RCP<MueLu::FactoryManagerBase const>) at MueLu_Hierarchy_def.hpp:387
#22  0x7BA213 in MueLu::HierarchyManager<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >::SetupHierarchy(MueLu::Hierarchy<double, int, int, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >&) const at MueLu_HierarchyManager.hpp:194 (discriminator 5)
#23  0x48D8D7 in TrilinosStem::solve(int, int, int, int, int, int, int, int, int, int, int*, double, double, double*, double*, double*) at TrilinosStem.C:474
#24  0x47EC5E in __tea_leaf_module_MOD_tea_leaf
#25  0x4839FF in diffuse_