Created
November 8, 2023 19:07
-
-
Save xmfan/a95d2144e7d27cc5df2ad64e2bf9ae82 to your computer and use it in GitHub Desktop.
python benchmarks/dynamo/torchbench.py --performance --cold-start-latency --training --amp --backend inductor --device cuda --print-memory --only=stable_diffusion_unet --ddp --multiprocess --batch_size=8 --disable-cudagraphs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Backend dynamo failed in warmup() | |
| Traceback (most recent call last): | |
| File "/home/xmfan/core/pytorch/benchmarks/dynamo/common.py", line 2604, in warmup | |
| fn(model, example_inputs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 410, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/xmfan/core/pytorch/benchmarks/dynamo/torchbench.py", line 512, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/home/xmfan/core/pytorch/benchmarks/dynamo/torchbench.py", line 513, in resume_in_forward_and_backward_pass | |
| self.optimizer_zero_grad(mod) | |
| File "/home/xmfan/core/pytorch/benchmarks/dynamo/torchbench.py", line 515, in resume_in_forward_and_backward_pass | |
| pred = mod(*cloned_inputs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1510, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1519, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1509, in forward | |
| else self._run_ddp_forward(*inputs, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1345, in _run_ddp_forward | |
| return self.module(*inputs, **kwargs) # type: ignore[index] | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1510, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1519, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/diffusers/models/unet_2d_condition.py", line 797, in forward | |
| def forward( | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1510, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1519, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 410, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/fx/graph_module.py", line 736, in call_wrapped | |
| return self._wrapped_call(self, *args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/fx/graph_module.py", line 315, in __call__ | |
| raise e | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/fx/graph_module.py", line 302, in __call__ | |
| return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1510, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1519, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "<eval_with_key>.2079", line 272, in forward | |
| submod_7 = self.compiled_submod_7(submod_6, l__self___down_blocks_2_resnets_0_conv2_weight, l__self___down_blocks_2_resnets_0_conv2_bias, getitem_15, l__self___down_blocks_2_resnets_0_conv_shortcut_weight, l__self___down_blocks_2_resnets_0_conv_shortcut_bias, l__self___down_blocks_2_attentions_0_proj_in_weight, l__self___down_blocks_2_attentions_0_proj_in_bias); submod_6 = l__self___down_blocks_2_resnets_0_conv2_weight = l__self___down_blocks_2_resnets_0_conv2_bias = l__self___down_blocks_2_resnets_0_conv_shortcut_weight = l__self___down_blocks_2_resonv2_bias = l__self___down_blocks_2_resnets_0_conv_shortcut_weight = l__self___down_blocks_2_resnets_0_conv_shortcut_bias = l__self___down_blocks_2_attentions_0_proj_in_weight = l__self___down_blocks_2_attentions_0_proj_in_bias = None | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1510, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1519, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 353, in forward | |
| x = self.submod(*args) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 410, in _fn | |
| return fn(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_dynamo/external_utils.py", line 17, in inner | |
| return fn(*args, **kwargs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 4851, in forward | |
| return compiled_fn(full_args) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1967, in g | |
| return f(*args) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 3053, in runtime_wrapper | |
| all_outs = call_func_with_args( | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1991, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1967, in g | |
| return f(*args) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/autograd/function.py", line 551, in apply | |
| return super().apply(*args, **kwargs) # type: ignore[misc] | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 3850, in forward | |
| fw_outs = call_func_with_args( | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1991, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 859, in __call__ | |
| return self.get_current_callable()(inputs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 673, in run | |
| return model(new_inputs) | |
| File "/home/xmfan/.conda/envs/oss/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 887, in _run_from_cache | |
| return compiled_graph.compiled_artifact(inputs) | |
| File "/tmp/tmpwzfm4yb5/fk/cfkraikmbglnom3b7uqjr4tnhxr6uldaw4lhjcxxcg5ktqkha4cp.py", line 482, in call | |
| assert_size_stride(primals_3, (8, 1280, 32, 32), (1310720, 1024, 32, 1)) | |
| AssertionError: expected size 1280==1280, stride 1==1024 at dim=1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment