Created
July 12, 2024 21:11
-
-
Save ManfeiBai/cf084d713bc4871d8e5f95fe7913e364 to your computer and use it in GitHub Desktop.
1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(torch310) root@b7b12c30e894:/pytorch/xla# PJRT_DEVICE=TPU python test_7665.py | |
body computation: !!!!!!!!! | |
HloModule PyLoweringContext.21.29, entry_computation_layout={((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}))->(s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0})} | |
%PyLoweringContext.7 (p0.10: s64[], p1.14: s64[1], p2.17: s64[1], p3.22: s64[1], UnusedArgumentsPlaceholder.26: s64[1]) -> (s64[], s64[], s64[1], s64[1], s64[1]) { | |
%UnusedArgumentsPlaceholder.26 = s64[1]{0} parameter(4) | |
%p0.10 = s64[] parameter(0) | |
%constant.9 = s64[] constant(1) | |
%constant.8 = s64[] constant(1) | |
%multiply.11 = s64[] multiply(s64[] %constant.9, s64[] %constant.8) | |
%subtract.12 = s64[] subtract(s64[] %p0.10, s64[] %multiply.11) | |
%constant.13 = s64[] constant(1) | |
%p1.14 = s64[1]{0} parameter(1) | |
%p2.17 = s64[1]{0} parameter(2) | |
%constant.16 = s64[] constant(1) | |
%constant.15 = s64[] constant(1) | |
%multiply.18 = s64[] multiply(s64[] %constant.16, s64[] %constant.15) | |
%broadcast.19 = s64[1]{0} broadcast(s64[] %multiply.18), dimensions={} | |
%add.20 = s64[1]{0} add(s64[1]{0} %p2.17, s64[1]{0} %broadcast.19) | |
%p3.22 = s64[1]{0} parameter(3) | |
%constant.21 = s64[] constant(1) | |
%multiply.23 = s64[] multiply(s64[] %constant.13, s64[] %constant.21) | |
%broadcast.24 = s64[1]{0} broadcast(s64[] %multiply.23), dimensions={} | |
%add.25 = s64[1]{0} add(s64[1]{0} %p3.22, s64[1]{0} %broadcast.24) | |
ROOT %tuple.27 = (s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) tuple(s64[] %subtract.12, s64[] %constant.13, s64[1]{0} %p1.14, s64[1]{0} %add.20, s64[1]{0} %add.25) | |
} | |
ENTRY %PyLoweringContext.21.29 (in.1: (s64[], s64[1], s64[1], s64[1], s64[1])) -> (s64[], s64[], s64[1], s64[1], s64[1]) { | |
%in.1 = (s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) parameter(0) | |
%get-tuple-element.2 = s64[] get-tuple-element((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=0 | |
%get-tuple-element.3 = s64[1]{0} get-tuple-element((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=1 | |
%get-tuple-element.4 = s64[1]{0} get-tuple-element((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=2 | |
%get-tuple-element.5 = s64[1]{0} get-tuple-element((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=3 | |
%get-tuple-element.6 = s64[1]{0} get-tuple-element((s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=4 | |
ROOT %call.28 = (s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) call(s64[] %get-tuple-element.2, s64[1]{0} %get-tuple-element.3, s64[1]{0} %get-tuple-element.4, s64[1]{0} %get-tuple-element.5, s64[1]{0} %get-tuple-element.6), to_apply=%PyLoweringContext.7 | |
} | |
cond computation: !!!!!!!!! | |
HloModule PyLoweringContext.8.16, entry_computation_layout={((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}))->pred[]} | |
%PyLoweringContext.7 (p0.9: s64[], UnusedArgumentsPlaceholder.11: s64[], UnusedArgumentsPlaceholder.12: s64[1], UnusedArgumentsPlaceholder.13: s64[1], UnusedArgumentsPlaceholder.14: s64[1]) -> pred[] { | |
%p0.9 = s64[] parameter(0) | |
%constant.8 = s64[] constant(0) | |
ROOT %compare.10 = pred[] compare(s64[] %p0.9, s64[] %constant.8), direction=GT | |
%UnusedArgumentsPlaceholder.11 = s64[] parameter(1) | |
%UnusedArgumentsPlaceholder.12 = s64[1]{0} parameter(2) | |
%UnusedArgumentsPlaceholder.13 = s64[1]{0} parameter(3) | |
%UnusedArgumentsPlaceholder.14 = s64[1]{0} parameter(4) | |
} | |
ENTRY %PyLoweringContext.8.16 (in.1: (s64[], s64[], s64[1], s64[1], s64[1])) -> pred[] { | |
%in.1 = (s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) parameter(0) | |
%get-tuple-element.2 = s64[] get-tuple-element((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=0 | |
%get-tuple-element.3 = s64[] get-tuple-element((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=1 | |
%get-tuple-element.4 = s64[1]{0} get-tuple-element((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=2 | |
%get-tuple-element.5 = s64[1]{0} get-tuple-element((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=3 | |
%get-tuple-element.6 = s64[1]{0} get-tuple-element((s64[], s64[], s64[1]{0}, s64[1]{0}, s64[1]{0}) %in.1), index=4 | |
ROOT %call.15 = pred[] call(s64[] %get-tuple-element.2, s64[] %get-tuple-element.3, s64[1]{0} %get-tuple-element.4, s64[1]{0} %get-tuple-element.5, s64[1]{0} %get-tuple-element.6), to_apply=%PyLoweringContext.7 | |
} | |
F0712 21:07:50.199203 1781370 debug_macros.h:21] Non-OK-status: status.status() | |
Status: INVALID_ARGUMENT: The parameter of condition and body, the result of the body, and init must all have the same shape; got Condition: (in: (s64[], s64[], s64[1], s64[1], s64[1])) -> pred[]; body: (in: (s64[], s64[1], s64[1], s64[1], s64[1])) -> (s64[], s64[], s64[1], s64[1], s64[1]); init: (s64[], s64[], s64[1], s64[1], s64[1])..: | |
*** Begin stack trace *** | |
tsl::CurrentStackTrace[abi:cxx11]() | |
xla::XlaComputation ConsumeValue<xla::XlaComputation>(absl::lts_20230802::StatusOr<xla::XlaComputation>&&) | |
_PyObject_MakeTpCall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyObject_FastCallDictTstate | |
_PyObject_Call_Prepend | |
_PyObject_MakeTpCall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyObject_FastCallDictTstate | |
_PyObject_Call_Prepend | |
PyObject_Call | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyObject_FastCallDictTstate | |
_PyObject_Call_Prepend | |
_PyObject_MakeTpCall | |
_PyEval_EvalFrameDefault | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
_PyFunction_Vectorcall | |
_PyEval_EvalFrameDefault | |
PyEval_EvalCode | |
_PyRun_SimpleFileObject | |
_PyRun_AnyFileObject | |
Py_RunMain | |
Py_BytesMain | |
__libc_start_main | |
*** End stack trace *** | |
*** Check failure stack trace: *** | |
@ 0x7f378051f0d9 absl::lts_20230802::log_internal::LogMessageFatal::~LogMessageFatal() | |
@ 0x7f37780455e1 ConsumeValue<>() | |
@ 0x7f37780a9f86 torch_xla::(anonymous namespace)::CreateComputation() | |
@ 0x7f37780aa3c4 pybind11::cpp_function::initialize<>()::{lambda()#3}::_FUN() | |
@ 0x7f3778077a76 pybind11::cpp_function::dispatcher() | |
@ 0x4fdc87 cfunction_call | |
https://symbolize.stripped_domain/r/?trace=7f3834662ce1,7f383496013f,7f378051e9bc,7f378051f0d8,7f37780455e0,7f37780a9f85,7f37780aa3c3,7f3778077a75,4fdc86&map= | |
*** SIGABRT received by PID 1781370 (TID 1781370) on cpu 153 from PID 1781370; stack trace: *** | |
PC: @ 0x7f3834662ce1 (unknown) raise | |
@ 0x7f3343003681 944 (unknown) | |
@ 0x7f3834960140 (unknown) (unknown) | |
@ 0x7f378051e9bd 32 absl::lts_20230802::log_internal::LogMessage::Die() | |
@ 0x7f378051f0d9 16 absl::lts_20230802::log_internal::LogMessageFatal::~LogMessageFatal() | |
@ 0x7f37780455e1 112 ConsumeValue<>() | |
@ 0x7f37780a9f86 672 torch_xla::(anonymous namespace)::CreateComputation() | |
@ 0x7f37780aa3c4 192 pybind11::cpp_function::initialize<>()::{lambda()#3}::_FUN() | |
@ 0x7f3778077a76 704 pybind11::cpp_function::dispatcher() | |
@ 0x4fdc87 (unknown) cfunction_call | |
@ ... and at least 2 more frames | |
https://symbolize.stripped_domain/r/?trace=7f3834662ce1,7f3343003680,7f383496013f,7f378051e9bc,7f378051f0d8,7f37780455e0,7f37780a9f85,7f37780aa3c3,7f3778077a75,4fdc86&map= | |
E0712 21:07:50.323291 1781370 coredump_hook.cc:316] RAW: Remote crash data gathering hook invoked. | |
E0712 21:07:50.323309 1781370 client.cc:269] RAW: Coroner client retries enabled, will retry for up to 30 sec. | |
E0712 21:07:50.323318 1781370 coredump_hook.cc:411] RAW: Sending fingerprint to remote end. | |
E0712 21:07:50.323340 1781370 coredump_hook.cc:420] RAW: Cannot send fingerprint to Coroner: [NOT_FOUND] stat failed on crash reporting socket /var/google/services/logmanagerd/remote_coredump.socket (Is the listener running?): No such file or directory | |
E0712 21:07:50.323346 1781370 coredump_hook.cc:472] RAW: Dumping core locally. | |
E0712 21:08:37.643836 1781370 process_state.cc:805] RAW: Raising signal 6 with default behavior | |
Aborted (core dumped) | |
(torch310) root@b7b12c30e894:/pytorch/xla# cat test_7665.py | |
import torch | |
import torch_xla | |
import torch_xla.experimental.fori_loop | |
from torch._higher_order_ops.while_loop import while_loop | |
import torch_xla.core.xla_model as xm | |
class TPUComputation: | |
def __init__(self): | |
self.device = xm.xla_device() | |
self.init_x = torch.tensor([1], device=self.device) | |
self.init_y = torch.tensor([1], device=self.device) | |
self.init_z = torch.tensor([1], device=self.device) | |
self.iteri = torch.tensor(10, device=self.device) | |
self.quantity = torch.tensor(1, device=self.device) | |
def cond_fn(self, iteri, x, y, z, q=None): | |
return iteri > 0 | |
def body_fn(self, iteri, x, y, z, q=None): | |
return iteri - 1, x.clone(), y.add(1), z + self.quantity # Problemmatic Line | |
def compute(self): | |
result = while_loop( | |
self.cond_fn, | |
self.body_fn, | |
(self.iteri, self.init_x, self.init_y, self.init_z), | |
) | |
return result | |
if __name__ == "__main__": | |
computation = TPUComputation() | |
result = computation.compute() | |
print(result) | |
(torch310) root@b7b12c30e894:/pytorch/xla# |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment