Created
June 2, 2023 20:20
-
-
Save trevor-m/d1d8912f8ab0d96da315a0f6e2f4aff5 to your computer and use it in GitHub Desktop.
PAXML PJRT Segfault asan
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| AddressSanitizer:DEADLYSIGNAL | |
| ================================================================= | |
| ==54208==ERROR: AddressSanitizer: SEGV on unknown address 0x00000000d3c0 (pc 0x7f8464007a7c bp 0x00000000d3c0 sp 0x6310013ea1f0 T0) | |
| ==54208==The signal is caused by a READ memory access. | |
| #0 0x7f8464007a7c in pthread_kill (/usr/lib/x86_64-linux-gnu/libc.so.6+0x96a7c) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) | |
| #1 0x7f8463fb3475 in gsignal (/usr/lib/x86_64-linux-gnu/libc.so.6+0x42475) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) | |
| #2 0x7f8463fb351f (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4251f) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) | |
| #3 0x7f8282aff8f6 in iree_hal_resource_set_free resource_set.c | |
| #4 0x7f8282afb705 in iree_hal_deferred_command_buffer_destroy deferred_command_buffer.c | |
| #5 0x7f828231da39 in iree_vm_ref_move ref.c | |
| #6 0x7f82822f2699 in iree_vm_bytecode_issue_import_call dispatch.c | |
| #7 0x7f82822f05c1 in iree_vm_bytecode_call_import dispatch.c | |
| #8 0x7f82822e41ad in iree_vm_bytecode_dispatch dispatch.c | |
| #9 0x7f82822db8c8 in iree_vm_bytecode_dispatch_begin dispatch.c | |
| #10 0x7f82822d9abc in iree_vm_bytecode_module_begin_call module.c | |
| #11 0x7f8282315ff7 in iree_vm_begin_invoke invocation.c | |
| #12 0x7f8282314f2b in iree_vm_invoke invocation.c | |
| #13 0x7f828229cd78 in iree::pjrt::LoadedExecutableInstance::BatchExecute(PJRT_LoadedExecutable_Execute_Args*) api_impl.cc | |
| #14 0x7f82822a8549 in iree::pjrt::LoadedExecutableInstance::BindApi(PJRT_Api*)::$_54::__invoke(PJRT_LoadedExecutable_Execute_Args*) api_impl.cc | |
| #15 0x7f845a2c48fa in xla::PjRtCApiLoadedExecutable::Execute(absl::lts_20230125::Span<std::vector<xla::PjRtBuffer*, std::allocator<xla::PjRtBuffer*> > const>, xla::ExecuteOptions const&, std::optional<std::vector<xla::PjRtFuture<absl::lts_20230125::Status>, std::allocator<xla::PjRtFuture<absl::lts_20230125::Status> > > >&) (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0xc838fa) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #16 0x7f845c885dd6 in xla::ifrt::PjRtLoadedExecutable::Execute(absl::lts_20230125::Span<tsl::RCReference<xla::ifrt::Array> >, xla::ExecuteOptions const&, std::optional<xla::ifrt::DeviceList>) (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0x3244dd6) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #17 0x7f845a273126 in absl::lts_20230125::StatusOr<xla::PyExecuteResults> xla::(anonymous namespace)::ExecuteShardedOnLocalDevicesInternal<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, xla::(anonymous namespace)::ShardedBufferAdapter<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >(xla::ExecuteOptions const&, std::shared_ptr<xla::PyClient> const&, xla::ifrt::LoadedExecutable*, absl::lts_20230125::Span<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > const>, std::optional<std::vector<xla::PjRtFuture<absl::lts_20230125::Status>, std::allocator<xla::PjRtFuture<absl::lts_20230125::Status> > > >&) py_executable.cc | |
| #18 0x7f845a27447d in xla::PyLoadedExecutable::ExecuteSharded(std::vector<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, std::allocator<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >, bool) (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0xc3347d) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #19 0x7f8459f68be3 in void pybind11::cpp_function::initialize<xla::ValueOrThrowWrapper<absl::lts_20230125::StatusOr<xla::PyExecuteResults> (std::vector<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, std::allocator<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >, bool), xla::PyLoadedExecutable>, xla::PyExecuteResults, xla::PyLoadedExecutable&, std::vector<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, std::allocator<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >, bool, pybind11::name, pybind11::is_method, pybind11::sibling, pybind11::arg, pybind11::arg_v>(xla::ValueOrThrowWrapper<absl::lts_20230125::StatusOr<xla::PyExecuteResults> (std::vector<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, std::allocator<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >, bool), xla::PyLoadedExecutable>&&, xla::PyExecuteResults (*)(xla::PyLoadedExecutable&, std::vector<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > >, std::allocator<std::variant<xla::PyArray, std::vector<xla::PyArray, std::allocator<xla::PyArray> > > > >, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&, pybind11::arg const&, pybind11::arg_v const&)::'lambda1'(pybind11::detail::function_call&)::operator()(pybind11::detail::function_call&) const (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0x927be3) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #20 0x7f8459f3d8e0 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0x8fc8e0) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #21 0x55e76416799d (/usr/bin/python3.10+0x15c99d) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #22 0x55e76415e4aa in _PyObject_MakeTpCall (/usr/bin/python3.10+0x1534aa) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #23 0x55e764175f0a (/usr/bin/python3.10+0x16af0a) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #24 0x55e764156461 in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x14b461) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #25 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #26 0x55e764152aef in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x147aef) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #27 0x55e76415d633 in _PyObject_FastCallDictTstate (/usr/bin/python3.10+0x152633) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #28 0x55e764172d10 in _PyObject_Call_Prepend (/usr/bin/python3.10+0x167d10) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #29 0x55e76429060f (/usr/bin/python3.10+0x28560f) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #30 0x55e76417687a in PyObject_Call (/usr/bin/python3.10+0x16b87a) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #31 0x55e764152aef in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x147aef) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #32 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #33 0x55e764152aef in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x147aef) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #34 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #35 0x7f845a16bb9c in jax::PmapFunction::Call(pybind11::handle, _object* const*, unsigned long, _object*) (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0xb2ab9c) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #36 0x7f845a16c39a in JaxPmapFunction_tp_vectorcall (/usr/local/lib/python3.10/dist-packages/jaxlib/xla_extension.so+0xb2b39a) (BuildId: 87a9c5e3db2565f8631e59fe5e690269) | |
| #37 0x55e764150784 in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x145784) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #38 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #39 0x55e764150784 in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x145784) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #40 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #41 0x55e7641508ca in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x1458ca) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #42 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #43 0x55e7641508ca in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x1458ca) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #44 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #45 0x55e7641768e1 in PyObject_Call (/usr/bin/python3.10+0x16b8e1) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #46 0x55e764152aef in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x147aef) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #47 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #48 0x55e7641508ca in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x1458ca) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #49 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #50 0x55e764151ade in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x146ade) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #51 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #52 0x55e764151ade in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x146ade) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #53 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #54 0x55e764151ade in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x146ade) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #55 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #56 0x55e764150784 in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x145784) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #57 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #58 0x55e764150784 in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x145784) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #59 0x55e7641681eb in _PyFunction_Vectorcall (/usr/bin/python3.10+0x15d1eb) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #60 0x55e764151ade in _PyEval_EvalFrameDefault (/usr/bin/python3.10+0x146ade) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #61 0x55e76414ced5 (/usr/bin/python3.10+0x141ed5) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #62 0x55e764243365 in PyEval_EvalCode (/usr/bin/python3.10+0x238365) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #63 0x55e764270107 (/usr/bin/python3.10+0x265107) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #64 0x55e764268f5a (/usr/bin/python3.10+0x25df5a) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #65 0x55e76426fe54 (/usr/bin/python3.10+0x264e54) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #66 0x55e76426f337 in _PyRun_SimpleFileObject (/usr/bin/python3.10+0x264337) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #67 0x55e76426f032 in _PyRun_AnyFileObject (/usr/bin/python3.10+0x264032) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #68 0x55e7642602dd in Py_RunMain (/usr/bin/python3.10+0x2552dd) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #69 0x55e76423632c in Py_BytesMain (/usr/bin/python3.10+0x22b32c) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| #70 0x7f8463f9ad8f (/usr/lib/x86_64-linux-gnu/libc.so.6+0x29d8f) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) | |
| #71 0x7f8463f9ae3f in __libc_start_main (/usr/lib/x86_64-linux-gnu/libc.so.6+0x29e3f) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) | |
| #72 0x55e764236224 in _start (/usr/bin/python3.10+0x22b224) (BuildId: 148e086667839ef13939196984d6f717c331bd76) | |
| AddressSanitizer can not provide additional info. | |
| SUMMARY: AddressSanitizer: SEGV (/usr/lib/x86_64-linux-gnu/libc.so.6+0x96a7c) (BuildId: 69389d485a9793dbe873f0ea2c93e02efaa9aa3d) in pthread_kill | |
| ==54208==ABORTING |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment