Created
July 2, 2024 05:47
-
-
Save shunting314/9b3a77ab8e4854dcbf217feacff56f05 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (pytorch) [shunting@devgpu005.nha1 ~/ws/pytorch (dash)]$ time python benchmarks/dynamo/torchbench.py --accuracy --inference --bfloat16 --backend inductor --device cuda --only moco --disable-cudagraphs | |
| loading model: 0it [00:00, ?it/s]NCCL version 2.18.5+cuda12.0 | |
| loading model: 0it [00:04, ?it/s] | |
| cuda eval moco | |
| [rank0]:W0701 22:41:14.175000 139849568687104 torch/_logging/_internal.py:1040] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored | |
| ERROR:common: | |
| Traceback (most recent call last): | |
| File "/home/shunting/ws/pytorch/benchmarks/dynamo/common.py", line 2636, in check_accuracy | |
| new_result = optimized_model_iter_fn(model_copy, example_inputs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/eval_frame.py", line 434, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 1121, in __call__ | |
| return self._torchdynamo_orig_callable( | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 948, in __call__ | |
| result = self._inner_convert( | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 472, in __call__ | |
| return _compile( | |
| File "/home/shunting/ws/pytorch/torch/_utils_internal.py", line 85, in wrapper_function | |
| return StrobelightCompileTimeProfiler.profile_compile_time( | |
| File "/home/shunting/ws/pytorch/torch/_strobelight/compile_time_profiler.py", line 129, in profile_compile_time | |
| return func(*args, **kwargs) | |
| File "/home/shunting/ws/miniconda3/envs/pytorch/lib/python3.10/contextlib.py", line 79, in inner | |
| return func(*args, **kwds) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 846, in _compile | |
| raise InternalTorchDynamoError(str(e)).with_traceback( | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 817, in _compile | |
| guarded_code = compile_inner(code, one_graph, hooks, transform) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/utils.py", line 233, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 636, in compile_inner | |
| out_code = transform_code_object(code, transform) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/bytecode_transformation.py", line 1270, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 178, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/convert_frame.py", line 582, in transform | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2476, in run | |
| super().run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1533, in CALL_FUNCTION_KW | |
| self.call_function(fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/lazy.py", line 132, in realize_and_forward | |
| return getattr(self.realize(), name)(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/user_defined.py", line 745, in call_function | |
| return func_var.call_function(tx, [obj_var] + args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 295, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 95, in call_function | |
| return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 760, in inline_user_function_return | |
| return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2691, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2807, in inline_call_ | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1521, in CALL_FUNCTION_EX | |
| self.call_function(fn, argsvars.items, kwargsvars) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/lazy.py", line 132, in realize_and_forward | |
| return getattr(self.realize(), name)(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/nn_module.py", line 875, in call_function | |
| return variables.UserFunctionVariable(fn, source=source).call_function( | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 295, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 95, in call_function | |
| return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 760, in inline_user_function_return | |
| return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2691, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2807, in inline_call_ | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1521, in CALL_FUNCTION_EX | |
| self.call_function(fn, argsvars.items, kwargsvars) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 353, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 295, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 95, in call_function | |
| return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 760, in inline_user_function_return | |
| return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2691, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2807, in inline_call_ | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1480, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 353, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 295, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 95, in call_function | |
| return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 760, in inline_user_function_return | |
| return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2691, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2807, in inline_call_ | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1480, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 963, in call_function | |
| return handler(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 838, in builtin_dipatch | |
| rv = handler(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 765, in call_self_handler | |
| result = self_handler(tx, *args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 1471, in call_hasattr | |
| return obj.call_hasattr(tx, name) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/user_defined.py", line 1048, in call_hasattr | |
| ).call_function(tx, [variables.ConstantVariable.create(name)], {}) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 353, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 295, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/functions.py", line 95, in call_function | |
| return tx.inline_user_function_return(self, [*self.self_args(), *args], kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 760, in inline_user_function_return | |
| return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2691, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 2807, in inline_call_ | |
| tracer.run() | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 904, in run | |
| while self.step(): | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 816, in step | |
| self.dispatch_table[inst.opcode](self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 510, in wrapper | |
| return inner_fn(self, inst) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 1480, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 963, in call_function | |
| return handler(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 712, in <lambda> | |
| return lambda tx, args, kwargs: obj.call_function( | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 963, in call_function | |
| return handler(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 847, in builtin_dipatch | |
| rv = fn(tx, args, kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 765, in call_self_handler | |
| result = self_handler(tx, *args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/builtin.py", line 1594, in call_getattr | |
| return obj.var_getattr(tx, name) | |
| File "/home/shunting/ws/pytorch/torch/_dynamo/variables/nn_module.py", line 269, in var_getattr | |
| subobj = inspect.getattr_static(base, name) | |
| File "/home/shunting/ws/miniconda3/envs/pytorch/lib/python3.10/inspect.py", line 1769, in getattr_static | |
| raise AttributeError(attr) | |
| torch._dynamo.exc.InternalTorchDynamoError: buffer_hook | |
| from user code: | |
| File "/home/shunting/ws/pytorch/benchmarks/dynamo/common.py", line 2365, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/home/shunting/ws/pytorch/benchmarks/dynamo/torchbench.py", line 448, in forward_pass | |
| return mod(*inputs) | |
| File "/home/shunting/ws/pytorch/torch/nn/parallel/distributed.py", line 1641, in forward | |
| inputs, kwargs = self._pre_forward(*inputs, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/nn/parallel/distributed.py", line 1536, in _pre_forward | |
| if self._check_sync_bufs_pre_fwd(): | |
| File "/home/shunting/ws/pytorch/torch/nn/parallel/distributed.py", line 2129, in _check_sync_bufs_pre_fwd | |
| not hasattr(self, "buffer_hook") | |
| File "/home/shunting/ws/pytorch/torch/nn/parallel/distributed.py", line 1244, in __getattr__ | |
| return getattr(self.module, name) | |
| Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| import torch._dynamo | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| fail_to_run | |
| [rank0]:[W701 22:41:14.640398764 ProcessGroupNCCL.cpp:1164] Warning: WARNING: process group has NOT been destroyed before we destruct ProcessGroupNCCL. On normal program exit, the application should call destroy_process_group to ensure that any pending NCCL operations have finished in this process. In rare cases this process can exit before this point and block the progress of another member of the process group. This constraint has always been present, but this warning has only been added since PyTorch 2.4 (function operator()) | |
| real 0m13.576s | |
| user 2m9.636s | |
| sys 0m22.125s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment