Created
November 20, 2022 21:35
-
-
Save ezyang/3b3be6de1670829b311f24e09f7cc76e to your computer and use it in GitHub Desktop.
Sweep logs for symbolic-shapes --accuracy --backend aot_eager --training (TORCHDYNAMO_DYNAMIC_SHAPES=1) - 54c29d7f15911f7bd62456353611b8eab4ca42dd Sun Nov 20 11:10:13 PST 2022
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Running torchbench.py BERT_pytorch... | |
| cuda train BERT_pytorch PASS | |
| Running torchbench.py Background_Matting... | |
| cuda train Background_Matting PASS | |
| WARNING:root:DALLE2_pytorch failed to load | |
| Eager model failed to run | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1013, in validate_model | |
| self.model_iter_fn(model, example_inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 338, in forward_and_backward_pass | |
| self.grad_scaler.scale(loss).backward() | |
| File "/data/users/ezyang/b/pytorch/torch/_tensor.py", line 473, in backward | |
| torch.autograd.backward( | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/__init__.py", line 197, in backward | |
| Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass | |
| RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1961, in run | |
| device, name, model, example_inputs, batch_size = runner.load_model( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 283, in load_model | |
| self.validate_model(model, example_inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1015, in validate_model | |
| raise NotImplementedError("Eager model failed to run") | |
| NotImplementedError: Eager model failed to run | |
| Running torchbench.py LearningToPaint... | |
| cuda train LearningToPaint PASS | |
| Running torchbench.py Super_SloMo... | |
| cuda train Super_SloMo FAIL (TIMEOUT) | |
| Running torchbench.py alexnet... | |
| cuda train alexnet PASS | |
| Running torchbench.py attention_is_all_you_need_pytorch... | |
| cuda train attention_is_all_you_need_pytorch PASS | |
| Running torchbench.py dcgan... | |
| cuda train dcgan PASS | |
| Running torchbench.py densenet121... | |
| cuda train densenet121 PASS | |
| WARNING:root:detectron2_fcos_r_50_fpn failed to load | |
| FCOS train is not supported by upstream detectron2. See GH Issue: https://github.com/facebookresearch/detectron2/issues/4369. | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1961, in run | |
| device, name, model, example_inputs, batch_size = runner.load_model( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 252, in load_model | |
| benchmark = benchmark_cls( | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/util/model.py", line 18, in __call__ | |
| obj = type.__call__(cls, *args, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/detectron2_fcos_r_50_fpn/__init__.py", line 15, in __init__ | |
| super().__init__(variant="COCO-Detection/fcos_R_50_FPN_1x.py", test=test, device=device, | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/util/framework/detectron2/model_factory.py", line 100, in __init__ | |
| loader = self.setup_train(cfg, args) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/util/framework/detectron2/model_factory.py", line 110, in setup_train | |
| raise NotImplementedError("FCOS train is not supported by upstream detectron2. " \ | |
| NotImplementedError: FCOS train is not supported by upstream detectron2. See GH Issue: https://github.com/facebookresearch/detectron2/issues/4369. | |
| WARNING:root:detectron2_maskrcnn_r_50_c4 failed to load | |
| Eager model failed to run | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1013, in validate_model | |
| self.model_iter_fn(model, example_inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 337, in forward_and_backward_pass | |
| loss = self.compute_loss(pred) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 327, in compute_loss | |
| return reduce_to_scalar_loss(pred) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/testing.py", line 99, in reduce_to_scalar_loss | |
| return sum([reduce_to_scalar_loss(x) for x in out]) / len(out) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/testing.py", line 99, in <listcomp> | |
| return sum([reduce_to_scalar_loss(x) for x in out]) / len(out) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/testing.py", line 109, in reduce_to_scalar_loss | |
| return sum([reduce_to_scalar_loss(value) for value in out.values()]) / len( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/testing.py", line 109, in <listcomp> | |
| return sum([reduce_to_scalar_loss(value) for value in out.values()]) / len( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/testing.py", line 114, in reduce_to_scalar_loss | |
| raise NotImplementedError("Don't know how to reduce", type(out)) | |
| NotImplementedError: ("Don't know how to reduce", <class 'detectron2.structures.instances.Instances'>) | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1961, in run | |
| device, name, model, example_inputs, batch_size = runner.load_model( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 283, in load_model | |
| self.validate_model(model, example_inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1015, in validate_model | |
| raise NotImplementedError("Eager model failed to run") | |
| NotImplementedError: Eager model failed to run | |
| Running torchbench.py dlrm... | |
| cuda train dlrm PASS | |
| /data/users/ezyang/b/pytorch/torch/utils/tensorboard/__init__.py:4: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. | |
| if not hasattr(tensorboard, "__version__") or LooseVersion( | |
| /home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/gym/core.py:317: DeprecationWarning: [33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.[0m | |
| deprecation( | |
| Running torchbench.py drq... | |
| cuda train drq PASS | |
| Running torchbench.py fastNLP_Bert... | |
| cuda train fastNLP_Bert PASS | |
| Running torchbench.py functorch_dp_cifar10... | |
| cuda train functorch_dp_cifar10 PASS | |
| Running torchbench.py functorch_maml_omniglot... | |
| cuda train functorch_maml_omniglot PASS | |
| Running torchbench.py hf_Albert... | |
| cuda train hf_Albert PASS | |
| Running torchbench.py hf_Bart... | |
| cuda train hf_Bart PASS | |
| Running torchbench.py hf_Bert... | |
| cuda train hf_Bert PASS | |
| Running torchbench.py hf_BigBird... | |
| ERROR:common:Output 0 of CompiledFunctionBackward is a view and is being modified inplace. This view was created inside a custom Function (or because an input was returned as-is) and the autograd logic to handle view+inplace would override the custom backward associated with the custom Function, leading to incorrect gradients. This behavior is forbidden. You can fix this by cloning the output of the custom Function. | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 2462, in forward | |
| outputs = self.bert( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 2148, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 1641, in forward | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 1493, in forward | |
| self_attention_outputs = self.attention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 1406, in forward | |
| self_outputs = self.self( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 475, in forward | |
| context_layer, attention_probs = self.bigbird_block_sparse_attention( | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 573, in bigbird_block_sparse_attention | |
| np.random.seed(seed) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/big_bird/modeling_big_bird.py", line 635, in <graph break in bigbird_block_sparse_attention> | |
| first_context_layer.unsqueeze_(2) | |
| RuntimeError: Output 0 of CompiledFunctionBackward is a view and is being modified inplace. This view was created inside a custom Function (or because an input was returned as-is) and the autograd logic to handle view+inplace would override the custom backward associated with the custom Function, leading to incorrect gradients. This behavior is forbidden. You can fix this by cloning the output of the custom Function. | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train hf_BigBird FAIL | |
| Running torchbench.py hf_DistilBert... | |
| cuda train hf_DistilBert PASS | |
| Running torchbench.py hf_GPT2... | |
| cuda train hf_GPT2 PASS | |
| Running torchbench.py hf_GPT2_large... | |
| cuda train hf_GPT2_large PASS | |
| Running torchbench.py hf_Longformer... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1813, in forward | |
| outputs = self.longformer( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1696, in forward | |
| padding_len, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds = self._pad_to_window_size( | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1715, in <graph break in forward> | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1265, in forward | |
| is_global_attn = is_index_global_attn.flatten().any().item() | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1297, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1221, in forward | |
| self_attn_outputs = self.attention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1157, in forward | |
| self_outputs = self.self( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 542, in forward | |
| def forward( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train hf_Longformer FAIL | |
| Running torchbench.py hf_Reformer... | |
| cuda train hf_Reformer PASS | |
| Running torchbench.py hf_T5... | |
| WARNING:common:fp64 golden ref were not generated for hf_T5 | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| RuntimeError: Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <built-in function iadd>(*(FakeTensor(FakeTensor(..., device='meta', size=(s2, s1, s0, s0), | |
| grad_fn=<AsStridedBackward0>), cuda:0), FakeTensor(FakeTensor(..., device='meta', size=(s2, s1, s0, s0), grad_fn=<AddBackward0>), cuda:0)), **{}): | |
| Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 41, in forward | |
| return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1601, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 945, in forward | |
| attention_mask = torch.ones(batch_size, mask_seq_length).to(inputs_embeds.device) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1033, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 664, in forward | |
| self_attention_outputs = self.layer[0]( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 570, in forward | |
| attention_output = self.SelfAttention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 519, in forward | |
| position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 120, in impl | |
| self.push(fn_var.call_function(self, self.popn(nargs), {})) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builtin.py", line 320, in call_function | |
| return wrap_fx_proxy(tx, proxy, **options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train hf_T5 FAIL | |
| Running torchbench.py hf_T5_base... | |
| WARNING:common:fp64 golden ref were not generated for hf_T5_base | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| RuntimeError: Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <built-in function iadd>(*(FakeTensor(FakeTensor(..., device='meta', size=(s2, s1, s0, s0), | |
| grad_fn=<AsStridedBackward0>), cuda:0), FakeTensor(FakeTensor(..., device='meta', size=(s2, s1, s0, s0), grad_fn=<AddBackward0>), cuda:0)), **{}): | |
| Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 41, in forward | |
| return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1601, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 945, in forward | |
| attention_mask = torch.ones(batch_size, mask_seq_length).to(inputs_embeds.device) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1033, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 664, in forward | |
| self_attention_outputs = self.layer[0]( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 570, in forward | |
| attention_output = self.SelfAttention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 519, in forward | |
| position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 120, in impl | |
| self.push(fn_var.call_function(self, self.popn(nargs), {})) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builtin.py", line 320, in call_function | |
| return wrap_fx_proxy(tx, proxy, **options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train hf_T5_base FAIL | |
| Running torchbench.py hf_T5_large... | |
| cuda train hf_T5_large PASS | |
| Running torchbench.py lennard_jones... | |
| cuda train lennard_jones PASS | |
| Running torchbench.py maml_omniglot... | |
| cuda train maml_omniglot PASS | |
| Running torchbench.py mnasnet1_0... | |
| cuda train mnasnet1_0 PASS | |
| Running torchbench.py mobilenet_v2... | |
| cuda train mobilenet_v2 PASS | |
| Running torchbench.py mobilenet_v2_quantized_qat... | |
| WARNING:common:fp64 golden ref were not generated for mobilenet_v2_quantized_qat | |
| ERROR:common:output 2: torch.Size([0]) != torch.Size([32]) | |
| While executing %_fused_moving_avg_obs_fq_helper_functional_1 : [#users=6] = call_function[target=torch.ops.aten._fused_moving_avg_obs_fq_helper_functional.default](args = (%mul : Tensor[size=[32, 3, 3, 3], stride=[27, 9, 3, 1]], %primals_170 : Tensor[size=[1], stride=[1]], %primals_169 : Tensor[size=[1], stride=[1]], %clone_6 : Tensor[size=[0], stride=[1]], %clone_7 : Tensor[size=[0], stride=[1]], %clone_4 : Tensor[size=[1], stride=[1]], %clone_5 : Tensor[size=[1], stride=[1]], 0.01, -128, 127, 0, True, True), kwargs = {}) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 660, in call_wrapped | |
| return self._wrapped_call(self, *args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 279, in __call__ | |
| raise e | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 269, in __call__ | |
| return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "<eval_with_key>.8", line 4, in forward | |
| def forward(self, x : torch.Tensor) -> torch.Tensor: | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 2: torch.Size([0]) != torch.Size([32]) | |
| While executing %_fused_moving_avg_obs_fq_helper_functional_1 : [#users=6] = call_function[target=torch.ops.aten._fused_moving_avg_obs_fq_helper_functional.default](args = (%mul : Tensor[size=[32, 3, 3, 3], stride=[27, 9, 3, 1]], %primals_170 : Tensor[size=[1], stride=[1]], %primals_169 : Tensor[size=[1], stride=[1]], %clone_6 : Tensor[size=[0], stride=[1]], %clone_7 : Tensor[size=[0], stride=[1]], %clone_4 : Tensor[size=[1], stride=[1]], %clone_5 : Tensor[size=[1], stride=[1]], 0.01, -128, 127, 0, True, True), kwargs = {}) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train mobilenet_v2_quantized_qat FAIL | |
| Running torchbench.py mobilenet_v3_large... | |
| cuda train mobilenet_v3_large PASS | |
| devgpu019:2092714:2092714 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 | |
| devgpu019:2092714:2092714 [0] NCCL INFO NCCL_SOCKET_IFNAME set to eth0 | |
| devgpu019:2092714:2092714 [0] NCCL INFO Bootstrap : Using eth0:2803:6080:6188:70b4::1<0> | |
| devgpu019:2092714:2092714 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation | |
| devgpu019:2092714:2092714 [0] NCCL INFO cudaDriverVersion 11040 | |
| NCCL version 2.14.3+cuda11.4 | |
| devgpu019:2092714:2093278 [0] NCCL INFO NCCL_IB_DISABLE set by environment to 1. | |
| devgpu019:2092714:2093278 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO NET/Socket : Using [0]eth0:2803:6080:6188:70b4::1<0> | |
| devgpu019:2092714:2093278 [0] NCCL INFO Using network Socket | |
| devgpu019:2092714:2093278 [0] NCCL INFO NET/Socket : GPU Direct RDMA Disabled for HCA 0 'eth0' | |
| devgpu019:2092714:2093278 [0] NCCL INFO === System : maxBw 5000.0 totalBw 0.0 === | |
| devgpu019:2092714:2093278 [0] NCCL INFO CPU/0 (1/1/2) | |
| devgpu019:2092714:2093278 [0] NCCL INFO + PCI[12.0] - PCI/D000 (11f840001d9bfbe1) | |
| devgpu019:2092714:2093278 [0] NCCL INFO + PCI[24.0] - PCI/F000 (11f840001d9bfbe0) | |
| devgpu019:2092714:2093278 [0] NCCL INFO + PCI[24.0] - GPU/11000 (0) | |
| devgpu019:2092714:2093278 [0] NCCL INFO + PCI[12.0] - NIC/30000 | |
| devgpu019:2092714:2093278 [0] NCCL INFO ========================================== | |
| devgpu019:2092714:2093278 [0] NCCL INFO GPU/11000 :GPU/11000 (0/5000.000000/LOC) CPU/0 (3/12.000000/PHB) | |
| devgpu019:2092714:2093278 [0] NCCL INFO Setting affinity for GPU 0 to ffffff,00000000,00000000,00ffffff | |
| devgpu019:2092714:2093278 [0] NCCL INFO Pattern 4, crossNic 0, nChannels 16, bw 44.000000/44.000000, type LOC/PIX, sameChannels 1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 0 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 1 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 2 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 3 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 4 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 5 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 6 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 7 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 8 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 9 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 10 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 11 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 12 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 13 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 14 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 15 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Pattern 3, crossNic 0, nChannels 16, bw 44.000000/44.000000, type LOC/PIX, sameChannels 1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 0 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 1 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 2 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 3 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 4 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 5 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 6 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 7 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 8 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 9 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 10 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 11 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 12 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 13 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 14 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 15 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Pattern 3, crossNic 0, nChannels 16, bw 44.000000/44.000000, type LOC/PIX, sameChannels 1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 0 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 1 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 2 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 3 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 4 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 5 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 6 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 7 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 8 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 9 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 10 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 11 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 12 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 13 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 14 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO 15 : GPU/0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 0 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 16 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 1 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 17 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 2 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 18 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 3 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 19 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 4 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 20 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 5 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 21 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 6 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 22 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 7 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 23 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 8 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 24 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 9 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 25 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 10 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 26 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 11 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 27 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 12 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 28 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 13 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 29 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 14 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 30 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 15 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Tree 31 : -1 -> 0 -> -1/-1/-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 00/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 01/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 02/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 03/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 04/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 05/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 06/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 07/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 08/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 09/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 10/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 11/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 12/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 13/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 14/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 15/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 16/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 17/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 18/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 19/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 20/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 21/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 22/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 23/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 24/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 25/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 26/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 27/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 28/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 29/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 30/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Channel 31/32 : 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 00 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 01 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 02 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 03 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 04 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 05 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 06 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 07 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 08 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 09 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 10 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 11 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 12 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 13 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 14 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 15 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 16 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 17 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 18 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 19 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 20 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 21 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 22 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 23 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 24 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 25 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 26 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 27 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 28 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 29 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 30 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Ring 31 : 0 -> 0 -> 0 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Trees [0] -1/-1/-1->0->-1 [1] -1/-1/-1->0->-1 [2] -1/-1/-1->0->-1 [3] -1/-1/-1->0->-1 [4] -1/-1/-1->0->-1 [5] -1/-1/-1->0->-1 [6] -1/-1/-1->0->-1 [7] -1/-1/-1->0->-1 [8] -1/-1/-1->0->-1 [9] -1/-1/-1->0->-1 [10] -1/-1/-1->0->-1 [11] -1/-1/-1->0->-1 [12] -1/-1/-1->0->-1 [13] -1/-1/-1->0->-1 [14] -1/-1/-1->0->-1 [15] -1/-1/-1->0->-1 [16] -1/-1/-1->0->-1 [17] -1/-1/-1->0->-1 [18] -1/-1/-1->0->-1 [19] -1/-1/-1->0->-1 [20] -1/-1/-1->0->-1 [21] -1/-1/-1->0->-1 [22] -1/-1/-1->0->-1 [23] -1/-1/-1->0->-1 [24] -1/-1/-1->0->-1 [25] -1/-1/-1->0->-1 [26] -1/-1/-1->0->-1 [27] -1/-1/-1->0->-1 [28] -1/-1/-1->0->-1 [29] -1/-1/-1->0->-1 [30] -1/-1/-1->0->-1 [31] -1/-1/-1->0->-1 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Connected all rings | |
| devgpu019:2092714:2093278 [0] NCCL INFO Connected all trees | |
| devgpu019:2092714:2093278 [0] NCCL INFO 32 coll channels, 32 p2p channels, 32 p2p channels per peer | |
| devgpu019:2092714:2093279 [0] NCCL INFO New proxy send connection 0 from local rank 0, transport 2 | |
| devgpu019:2092714:2093278 [0] NCCL INFO Connection to proxy localRank 0 -> connection 0x7fa910002e80 | |
| devgpu019:2092714:2093278 [0] NCCL INFO comm 0x6ae06c0 rank 0 nranks 1 cudaDev 0 busId 11000 - Init COMPLETE | |
| Running torchbench.py moco... | |
| ERROR:common: | |
| from user code: | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/moco/moco/builder.py", line 172, in concat_all_gather | |
| torch.distributed.all_gather(tensors_gather, tensor, async_op=False) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/distributed/distributed_c10d.py", line 1346, in wrapper | |
| return func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/distributed/distributed_c10d.py", line 2341, in all_gather | |
| work = default_pg.allgather([tensor_list], [tensor]) | |
| File "/data/users/ezyang/b/pytorch/torch/_subclasses/fake_tensor.py", line 875, in __torch_dispatch__ | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_ops.py", line 297, in __call__ | |
| return self._op(*args, **kwargs or {}) | |
| RuntimeError: Tensors must be CUDA and dense | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <function all_gather at 0x7fb232f7fee0>(*([FakeTensor(FakeTensor(..., device='meta', size=(s0, s1, s2, s2)), cuda:0)], FakeTensor(FakeTensor(..., device='meta', size=(s0, s1, s2, s2)), cuda:0)), **{'async_op': False}): | |
| Tensors must be CUDA and dense | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/parallel/distributed.py", line 1098, in forward | |
| output = self._run_ddp_forward(*inputs, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/parallel/distributed.py", line 1051, in _run_ddp_forward | |
| return module_to_run(*inputs[0], **kwargs[0]) # type: ignore[index] | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/moco/moco/builder.py", line 130, in forward | |
| self._momentum_update_key_encoder() # update the key encoder | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/moco/moco/builder.py", line 133, in <graph break in forward> | |
| im_k, idx_unshuffle = self._batch_shuffle_ddp(im_k) | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/grad_mode.py", line 27, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/moco/moco/builder.py", line 76, in _batch_shuffle_ddp | |
| x_gather = concat_all_gather(x) | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/grad_mode.py", line 27, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 959, in CALL_FUNCTION_KW | |
| self.call_function(fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/torch.py", line 430, in call_function | |
| tensor_variable = wrap_fx_proxy( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/moco/moco/builder.py", line 172, in concat_all_gather | |
| torch.distributed.all_gather(tensors_gather, tensor, async_op=False) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train moco FAIL | |
| devgpu019:2092714:2093279 [0] NCCL INFO [Service thread] Connection closed by localRank 0 | |
| devgpu019:2092714:2092714 [0] NCCL INFO comm 0x6ae06c0 rank 0 nranks 1 cudaDev 0 busId 11000 - Abort COMPLETE | |
| Running torchbench.py nvidia_deeprecommender... | |
| cuda train nvidia_deeprecommender PASS | |
| Running torchbench.py pytorch_CycleGAN_and_pix2pix... | |
| --dataroot /data/users/ezyang/b/torchbenchmark/torchbenchmark/data/.data/pytorch_CycleGAN_and_pix2pix_inputs/datasets/horse2zebra --name horse2zebra --model cycle_gan --display_id 0 --n_epochs 3 --n_epochs_decay 3 --gpu_ids 0 --checkpoints_dir /data/users/ezyang/b/torchbenchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/.data/checkpoints | |
| cuda train pytorch_CycleGAN_and_pix2pix PASS | |
| Running torchbench.py pytorch_stargan... | |
| cuda train pytorch_stargan PASS | |
| downloading en-ud-v2.zip | |
| 0%| | 0.00/688k [00:00<?, ?B/s] 7%|▋ | 49.2k/688k [00:00<00:01, 484kB/s] 14%|█▍ | 98.3k/688k [00:00<00:01, 395kB/s] 24%|██▍ | 164k/688k [00:00<00:01, 489kB/s] 33%|███▎ | 229k/688k [00:00<00:00, 539kB/s] 43%|████▎ | 295k/688k [00:00<00:00, 567kB/s] 55%|█████▍ | 377k/688k [00:00<00:00, 637kB/s] 64%|██████▍ | 442k/688k [00:00<00:00, 632kB/s] 79%|███████▊ | 541k/688k [00:00<00:00, 727kB/s] 93%|█████████▎| 639k/688k [00:00<00:00, 790kB/s] 100%|██████████| 688k/688k [00:01<00:00, 676kB/s] | |
| Running torchbench.py pytorch_struct... | |
| extracting | |
| cuda train pytorch_struct PASS | |
| Running torchbench.py pytorch_unet... | |
| cuda train pytorch_unet PASS | |
| Running torchbench.py resnet152... | |
| cuda train resnet152 PASS | |
| Running torchbench.py resnet18... | |
| cuda train resnet18 PASS | |
| Running torchbench.py resnet50... | |
| cuda train resnet50 PASS | |
| Running torchbench.py resnet50_quantized_qat... | |
| WARNING:common:fp64 golden ref were not generated for resnet50_quantized_qat | |
| ERROR:common:output 2: torch.Size([0]) != torch.Size([64]) | |
| While executing %_fused_moving_avg_obs_fq_helper_functional_1 : [#users=6] = call_function[target=torch.ops.aten._fused_moving_avg_obs_fq_helper_functional.default](args = (%mul : Tensor[size=[64, 3, 7, 7], stride=[147, 49, 7, 1]], %primals_173 : Tensor[size=[1], stride=[1]], %primals_172 : Tensor[size=[1], stride=[1]], %clone_6 : Tensor[size=[0], stride=[1]], %clone_7 : Tensor[size=[0], stride=[1]], %clone_4 : Tensor[size=[1], stride=[1]], %clone_5 : Tensor[size=[1], stride=[1]], 0.01, -128, 127, 0, True, True), kwargs = {}) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 660, in call_wrapped | |
| return self._wrapped_call(self, *args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 279, in __call__ | |
| raise e | |
| File "/data/users/ezyang/b/pytorch/torch/fx/graph_module.py", line 269, in __call__ | |
| return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "<eval_with_key>.8", line 4, in forward | |
| def forward(self, x : torch.Tensor) -> torch.Tensor: | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 2: torch.Size([0]) != torch.Size([64]) | |
| While executing %_fused_moving_avg_obs_fq_helper_functional_1 : [#users=6] = call_function[target=torch.ops.aten._fused_moving_avg_obs_fq_helper_functional.default](args = (%mul : Tensor[size=[64, 3, 7, 7], stride=[147, 49, 7, 1]], %primals_173 : Tensor[size=[1], stride=[1]], %primals_172 : Tensor[size=[1], stride=[1]], %clone_6 : Tensor[size=[0], stride=[1]], %clone_7 : Tensor[size=[0], stride=[1]], %clone_4 : Tensor[size=[1], stride=[1]], %clone_5 : Tensor[size=[1], stride=[1]], 0.01, -128, 127, 0, True, True), kwargs = {}) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train resnet50_quantized_qat FAIL | |
| Running torchbench.py resnext50_32x4d... | |
| cuda train resnext50_32x4d PASS | |
| Running torchbench.py shufflenet_v2_x1_0... | |
| cuda train shufflenet_v2_x1_0 PASS | |
| /home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/gym/core.py:317: DeprecationWarning: [33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.[0m | |
| deprecation( | |
| /home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: [33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.[0m | |
| deprecation( | |
| Running torchbench.py soft_actor_critic... | |
| cuda train soft_actor_critic PASS | |
| Running torchbench.py speech_transformer... | |
| ERROR:common:compile_fn raised AssertionError: While executing %self_layer_stack_0_slf_attn_attention_temperature : torch.Tensor [#users=1] = placeholder[target=self_layer_stack_0_slf_attn_attention_temperature] | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/output_graph.py", line 466, in call_user_compiler | |
| compiled_fn = self.compiler_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/debug_utils.py", line 865, in debug_wrapper | |
| compiled_gm = compiler_fn(gm, example_inputs, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/training.py", line 94, in compile_fn | |
| return cls(gm, example_inputs, fake_mode).verified_candidate() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/training.py", line 117, in __init__ | |
| if not is_aot_autograd_safe_to_run(gm, example_inputs, fake_mode): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/training.py", line 64, in is_aot_autograd_safe_to_run | |
| mutated = has_mutation(gm, example_inputs, fake_mode, inputs_only=True) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/analysis.py", line 138, in has_mutation | |
| ShapeAliasingAndMutationProp(new_gm).run(*example_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/analysis.py", line 116, in run | |
| super().run(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/analysis.py", line 50, in run_node | |
| result = getattr(self, n.op)(n.target, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/optimizations/analysis.py", line 41, in placeholder | |
| assert isinstance(value, torch.Tensor) | |
| AssertionError: While executing %self_layer_stack_0_slf_attn_attention_temperature : torch.Tensor [#users=1] = placeholder[target=self_layer_stack_0_slf_attn_attention_temperature] | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/speech_transformer/speech_transformer/transformer/transformer.py", line 28, in forward | |
| encoder_padded_outputs, *_ = self.encoder(padded_input, input_lengths) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/speech_transformer/speech_transformer/transformer/encoder.py", line 48, in forward | |
| non_pad_mask = get_non_pad_mask(padded_input, input_lengths=input_lengths) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/speech_transformer/speech_transformer/transformer/encoder.py", line 50, in <graph break in forward> | |
| slf_attn_mask = get_attn_pad_mask(padded_input, input_lengths, length) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/speech_transformer/speech_transformer/transformer/encoder.py", line 55, in <graph break in forward> | |
| self.positional_encoding(padded_input)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1679, in RETURN_VALUE | |
| self.output.compile_subgraph(self) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/output_graph.py", line 386, in compile_subgraph | |
| self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/output_graph.py", line 432, in compile_and_call_fx_graph | |
| compiled_fn = self.call_user_compiler(gm) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/output_graph.py", line 475, in call_user_compiler | |
| raise BackendCompilerFailed(self.compiler_fn, e) from e | |
| torch._dynamo.exc.BackendCompilerFailed: compile_fn raised AssertionError: While executing %self_layer_stack_0_slf_attn_attention_temperature : torch.Tensor [#users=1] = placeholder[target=self_layer_stack_0_slf_attn_attention_temperature] | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train speech_transformer FAIL | |
| Running torchbench.py squeezenet1_1... | |
| cuda train squeezenet1_1 PASS | |
| Running torchbench.py tacotron2... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tacotron2/model.py", line 505, in forward | |
| encoder_outputs = self.encoder(embedded_inputs, text_lengths) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tacotron2/model.py", line 507, in <graph break in forward> | |
| mel_outputs, gate_outputs, alignments = self.decoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tacotron2/model.py", line 396, in forward | |
| decoder_input = self.get_go_frame(memory).unsqueeze(0) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tacotron2/model.py", line 396, in <graph break in forward> | |
| decoder_input = self.get_go_frame(memory).unsqueeze(0) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train tacotron2 FAIL | |
| Running torchbench.py timm_efficientdet... | |
| ERROR:common:output 0: torch.Size([96, 1, 3, 3]) != torch.Size([144, 1, 5, 5]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/effdet/bench.py", line 133, in forward | |
| class_out, box_out = self.model(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/effdet/efficientdet.py", line 602, in forward | |
| x = self.backbone(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 604, in forward | |
| x = self.conv_stem(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 611, in <graph break in forward> | |
| x = b(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet_blocks.py", line 183, in forward | |
| x = self.conv_dw(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 30, in forward | |
| return conv2d_same(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 13, in conv2d_same | |
| def conv2d_same( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([96, 1, 3, 3]) != torch.Size([144, 1, 5, 5]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train timm_efficientdet FAIL | |
| Running torchbench.py timm_efficientnet... | |
| cuda train timm_efficientnet PASS | |
| Running torchbench.py timm_regnet... | |
| cuda train timm_regnet PASS | |
| Running torchbench.py timm_resnest... | |
| cuda train timm_resnest PASS | |
| Running torchbench.py timm_vision_transformer... | |
| cuda train timm_vision_transformer PASS | |
| Running torchbench.py timm_vision_transformer_large... | |
| cuda train timm_vision_transformer_large PASS | |
| Running torchbench.py timm_vovnet... | |
| cuda train timm_vovnet PASS | |
| Running torchbench.py tts_angular... | |
| ERROR:common: | |
| from user code: | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tts_angular/model.py", line 18, in <graph break in forward> | |
| o, (_, _) = self.lstm(x) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1096, in run_node | |
| return nnmodule(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/rnn.py", line 776, in forward | |
| result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers, | |
| RuntimeError: Cannot call sizes() on tensor with symbolic sizes/strides | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_module self_lstm(*(FakeTensor(FakeTensor(..., device='meta', size=(s0, s1, 40)), cuda:0),), **{}): | |
| Cannot call sizes() on tensor with symbolic sizes/strides | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tts_angular/model.py", line 59, in forward | |
| d = self.layers(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tts_angular/model.py", line 17, in forward | |
| self.lstm.flatten_parameters() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 910, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/nn_module.py", line 202, in call_function | |
| return wrap_fx_proxy( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/data/users/ezyang/b/torchbenchmark/torchbenchmark/models/tts_angular/model.py", line 18, in <graph break in forward> | |
| o, (_, _) = self.lstm(x) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train tts_angular FAIL | |
| Running torchbench.py vgg16... | |
| cuda train vgg16 PASS | |
| Running torchbench.py vision_maskrcnn... | |
| ERROR:common:output 0: torch.Size([3, 427, 640]) != torch.Size([3, 612, 612]) | |
| While executing %arg0_1 : [#users=1] = placeholder[target=arg0_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 333, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/torchbench.py", line 336, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchvision/torchvision/models/detection/generalized_rcnn.py", line 83, in forward | |
| images, targets = self.transform(images, targets) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/torchvision/torchvision/models/detection/transform.py", line 129, in forward | |
| image = self.normalize(image) | |
| File "/data/users/ezyang/b/torchvision/torchvision/models/detection/transform.py", line 148, in normalize | |
| def normalize(self, image: Tensor) -> Tensor: | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 639, in new_fn | |
| return call_func_with_args(compiled_fw, args, disable_amp=disable_amp) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([3, 427, 640]) != torch.Size([3, 612, 612]) | |
| While executing %arg0_1 : [#users=1] = placeholder[target=arg0_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train vision_maskrcnn FAIL | |
| Running torchbench.py yolov3... | |
| [2022-11-20 11:54:00,366] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) | |
| function: 'forward' (/data/users/ezyang/b/pytorch/torch/nn/modules/container.py:202) | |
| reasons: ['___check_obj_id(self, 140486728199232)'] | |
| to diagnose recompilation issues, see https://github.com/pytorch/torchdynamo/blob/main/TROUBLESHOOTING.md. | |
| cuda train yolov3 PASS | |
| Running huggingface.py AlbertForMaskedLM... | |
| cuda train AlbertForMaskedLM PASS | |
| Running huggingface.py AlbertForQuestionAnswering... | |
| cuda train AlbertForQuestionAnswering PASS | |
| Running huggingface.py AllenaiLongformerBase... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1813, in forward | |
| outputs = self.longformer( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1696, in forward | |
| padding_len, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds = self._pad_to_window_size( | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1715, in <graph break in forward> | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1265, in forward | |
| is_global_attn = is_index_global_attn.flatten().any().item() | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1297, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1221, in forward | |
| self_attn_outputs = self.attention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 1157, in forward | |
| self_outputs = self.self( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/longformer/modeling_longformer.py", line 542, in forward | |
| def forward( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train AllenaiLongformerBase FAIL | |
| Running huggingface.py BartForCausalLM... | |
| cuda train BartForCausalLM PASS | |
| Running huggingface.py BartForConditionalGeneration... | |
| cuda train BartForConditionalGeneration PASS | |
| Running huggingface.py BertForMaskedLM... | |
| cuda train BertForMaskedLM PASS | |
| Running huggingface.py BertForQuestionAnswering... | |
| cuda train BertForQuestionAnswering PASS | |
| Running huggingface.py BlenderbotForCausalLM... | |
| cuda train BlenderbotForCausalLM PASS | |
| Running huggingface.py BlenderbotSmallForCausalLM... | |
| cuda train BlenderbotSmallForCausalLM PASS | |
| Running huggingface.py BlenderbotSmallForConditionalGeneration... | |
| cuda train BlenderbotSmallForConditionalGeneration PASS | |
| Running huggingface.py CamemBert... | |
| cuda train CamemBert PASS | |
| Running huggingface.py DebertaForMaskedLM... | |
| cuda train DebertaForMaskedLM PASS | |
| Running huggingface.py DebertaForQuestionAnswering... | |
| cuda train DebertaForQuestionAnswering PASS | |
| Running huggingface.py DebertaV2ForMaskedLM... | |
| cuda train DebertaV2ForMaskedLM PASS | |
| Running huggingface.py DebertaV2ForQuestionAnswering... | |
| cuda train DebertaV2ForQuestionAnswering PASS | |
| WARNING:__main__:Sequence Length not defined for DistilBertForMaskedLM. Choosing 128 arbitrarily | |
| Running huggingface.py DistilBertForMaskedLM... | |
| cuda train DistilBertForMaskedLM PASS | |
| WARNING:__main__:Sequence Length not defined for DistilBertForQuestionAnswering. Choosing 128 arbitrarily | |
| Running huggingface.py DistilBertForQuestionAnswering... | |
| cuda train DistilBertForQuestionAnswering PASS | |
| Running huggingface.py DistillGPT2... | |
| cuda train DistillGPT2 PASS | |
| If you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.` | |
| Running huggingface.py ElectraForCausalLM... | |
| cuda train ElectraForCausalLM PASS | |
| Running huggingface.py ElectraForQuestionAnswering... | |
| cuda train ElectraForQuestionAnswering PASS | |
| Running huggingface.py GPT2ForSequenceClassification... | |
| cuda train GPT2ForSequenceClassification PASS | |
| Running huggingface.py GoogleFnet... | |
| cuda train GoogleFnet PASS | |
| Running huggingface.py LayoutLMForMaskedLM... | |
| cuda train LayoutLMForMaskedLM PASS | |
| Running huggingface.py LayoutLMForSequenceClassification... | |
| cuda train LayoutLMForSequenceClassification PASS | |
| WARNING:__main__:Sequence Length not defined for M2M100ForConditionalGeneration. Choosing 128 arbitrarily | |
| Running huggingface.py M2M100ForConditionalGeneration... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/m2m_100/modeling_m2m_100.py", line 1317, in forward | |
| outputs = self.model( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/m2m_100/modeling_m2m_100.py", line 1190, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/m2m_100/modeling_m2m_100.py", line 716, in forward | |
| def forward( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train M2M100ForConditionalGeneration FAIL | |
| Running huggingface.py MBartForCausalLM... | |
| cuda train MBartForCausalLM PASS | |
| Running huggingface.py MBartForConditionalGeneration... | |
| cuda train MBartForConditionalGeneration PASS | |
| WARNING:__main__:Sequence Length not defined for MT5ForConditionalGeneration. Choosing 128 arbitrarily | |
| Running huggingface.py MT5ForConditionalGeneration... | |
| WARNING:common:fp64 golden ref were not generated for MT5ForConditionalGeneration | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| RuntimeError: Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <built-in function iadd>(*(FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), | |
| grad_fn=<AsStridedBackward0>), cuda:0), FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), grad_fn=<AddBackward0>), cuda:0)), **{}): | |
| Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1601, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 945, in forward | |
| attention_mask = torch.ones(batch_size, mask_seq_length).to(inputs_embeds.device) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1033, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 664, in forward | |
| self_attention_outputs = self.layer[0]( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 570, in forward | |
| attention_output = self.SelfAttention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 519, in forward | |
| position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 120, in impl | |
| self.push(fn_var.call_function(self, self.popn(nargs), {})) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builtin.py", line 320, in call_function | |
| return wrap_fx_proxy(tx, proxy, **options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train MT5ForConditionalGeneration FAIL | |
| If you want to use `MegatronBertForCausalLM` as a standalone, add `is_decoder=True.` | |
| Running huggingface.py MegatronBertForCausalLM... | |
| cuda train MegatronBertForCausalLM PASS | |
| Running huggingface.py MegatronBertForQuestionAnswering... | |
| cuda train MegatronBertForQuestionAnswering PASS | |
| Running huggingface.py MobileBertForMaskedLM... | |
| cuda train MobileBertForMaskedLM PASS | |
| Running huggingface.py MobileBertForQuestionAnswering... | |
| cuda train MobileBertForQuestionAnswering PASS | |
| Running huggingface.py OPTForCausalLM... | |
| cuda train OPTForCausalLM PASS | |
| Running huggingface.py PLBartForCausalLM... | |
| cuda train PLBartForCausalLM PASS | |
| Running huggingface.py PLBartForConditionalGeneration... | |
| cuda train PLBartForConditionalGeneration PASS | |
| WARNING:__main__:Sequence Length not defined for PegasusForCausalLM. Choosing 128 arbitrarily | |
| Running huggingface.py PegasusForCausalLM... | |
| cuda train PegasusForCausalLM PASS | |
| WARNING:__main__:Sequence Length not defined for PegasusForConditionalGeneration. Choosing 128 arbitrarily | |
| Running huggingface.py PegasusForConditionalGeneration... | |
| cuda train PegasusForConditionalGeneration PASS | |
| If you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.` | |
| Running huggingface.py RobertaForCausalLM... | |
| cuda train RobertaForCausalLM PASS | |
| Running huggingface.py RobertaForQuestionAnswering... | |
| cuda train RobertaForQuestionAnswering PASS | |
| WARNING:__main__:Sequence Length not defined for Speech2Text2ForCausalLM. Choosing 128 arbitrarily | |
| Running huggingface.py Speech2Text2ForCausalLM... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py", line 910, in forward | |
| outputs = self.model.decoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py", line 509, in forward | |
| def forward( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train Speech2Text2ForCausalLM FAIL | |
| Running huggingface.py T5ForConditionalGeneration... | |
| WARNING:common:fp64 golden ref were not generated for T5ForConditionalGeneration | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| RuntimeError: Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <built-in function iadd>(*(FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), | |
| grad_fn=<AsStridedBackward0>), cuda:0), FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), grad_fn=<AddBackward0>), cuda:0)), **{}): | |
| Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1601, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 945, in forward | |
| attention_mask = torch.ones(batch_size, mask_seq_length).to(inputs_embeds.device) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1033, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 664, in forward | |
| self_attention_outputs = self.layer[0]( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 570, in forward | |
| attention_output = self.SelfAttention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 519, in forward | |
| position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 120, in impl | |
| self.push(fn_var.call_function(self, self.popn(nargs), {})) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builtin.py", line 320, in call_function | |
| return wrap_fx_proxy(tx, proxy, **options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train T5ForConditionalGeneration FAIL | |
| Running huggingface.py T5Small... | |
| WARNING:common:fp64 golden ref were not generated for T5Small | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| RuntimeError: Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <built-in function iadd>(*(FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), | |
| grad_fn=<AsStridedBackward0>), cuda:0), FakeTensor(FakeTensor(..., device='meta', size=(1, s1, s0, s0), grad_fn=<AddBackward0>), cuda:0)), **{}): | |
| Output 0 of AsStridedBackward0 is a view of a view which was created in no_grad mode and is being modified inplace with grad mode enabled. Given that this use case is ambiguous and error-prone, it is forbidden. You can clarify your code by moving both the view and the inplace either both inside the no_grad block (if you don't want the inplace to be tracked) or both outside (if you want the inplace to be tracked). | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1601, in forward | |
| encoder_outputs = self.encoder( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 945, in forward | |
| attention_mask = torch.ones(batch_size, mask_seq_length).to(inputs_embeds.device) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 1033, in <graph break in forward> | |
| layer_outputs = layer_module( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 664, in forward | |
| self_attention_outputs = self.layer[0]( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 570, in forward | |
| attention_output = self.SelfAttention( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 519, in forward | |
| position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 120, in impl | |
| self.push(fn_var.call_function(self, self.popn(nargs), {})) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builtin.py", line 320, in call_function | |
| return wrap_fx_proxy(tx, proxy, **options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py", line 529, in <graph break in forward> | |
| scores += position_bias | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train T5Small FAIL | |
| Running huggingface.py TrOCRForCausalLM... | |
| cuda train TrOCRForCausalLM PASS | |
| WARNING:__main__:Sequence Length not defined for XGLMForCausalLM. Choosing 128 arbitrarily | |
| Running huggingface.py XGLMForCausalLM... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xglm/modeling_xglm.py", line 889, in forward | |
| outputs = self.model( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xglm/modeling_xglm.py", line 591, in forward | |
| @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train XGLMForCausalLM FAIL | |
| Running huggingface.py XLNetLMHeadModel... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 482, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/huggingface.py", line 485, in <graph break in forward_and_backward_pass> | |
| pred = mod(**cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xlnet/modeling_xlnet.py", line 1448, in forward | |
| transformer_outputs = self.transformer( | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xlnet/modeling_xlnet.py", line 1207, in forward | |
| pos_emb = self.relative_positional_encoding(qlen, klen, bsz=bsz) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xlnet/modeling_xlnet.py", line 1237, in <graph break in forward> | |
| new_mems = new_mems + (self.cache_mem(output_h, mems[i]),) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/transformers/models/xlnet/modeling_xlnet.py", line 991, in cache_mem | |
| def cache_mem(self, curr_out, prev_mem): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train XLNetLMHeadModel FAIL | |
| Running huggingface.py YituTechConvBert... | |
| cuda train YituTechConvBert PASS | |
| Running timm_models.py adv_inception_v3... | |
| cuda train adv_inception_v3 PASS | |
| Running timm_models.py beit_base_patch16_224... | |
| cuda train beit_base_patch16_224 PASS | |
| Running timm_models.py botnet26t_256... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1559, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1551, in forward_features | |
| x = self.stages(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1245, in forward | |
| x = self.self_attn(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 137, in forward | |
| _assert(H == self.pos_embed.height, '') | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 152, in <graph break in forward> | |
| attn = (q @ k) * self.scale + self.pos_embed(q) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 68, in forward | |
| def forward(self, q): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train botnet26t_256 FAIL | |
| Running timm_models.py cait_m36_384... | |
| cuda train cait_m36_384 PASS | |
| Running timm_models.py coat_lite_mini... | |
| cuda train coat_lite_mini PASS | |
| Running timm_models.py convit_base... | |
| WARNING:common:fp64 golden ref were not generated for convit_base | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 138, in get_rel_indices | |
| indy = ind.repeat_interleave(img_size, dim=0).repeat_interleave(img_size, dim=1) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1093, in run_node | |
| return getattr(args[0], node.target)(*args[1:], **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_subclasses/fake_tensor.py", line 867, in __torch_dispatch__ | |
| op_impl_out = op_impl(self, func, *args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_subclasses/fake_tensor.py", line 362, in dyn_shape | |
| raise DynamicOutputShapeException(func) | |
| torch._subclasses.fake_tensor.DynamicOutputShapeException: aten.repeat_interleave.Tensor | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_method repeat_interleave(*(FakeTensor(FakeTensor(..., device='meta', size=(14, 14), dtype=torch.int64), cpu), 14), **{'dim': 0}): | |
| aten.repeat_interleave.Tensor | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 333, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 315, in forward_features | |
| x = self.patch_embed(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 323, in <graph break in forward_features> | |
| x = blk(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 214, in forward | |
| x = x + self.drop_path(self.attn(self.norm1(x))) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 86, in forward | |
| self.rel_indices = self.get_rel_indices(N) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 959, in CALL_FUNCTION_KW | |
| self.call_function(fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/misc.py", line 582, in call_function | |
| return self.obj.call_method(tx, self.name, args, kwargs).add_options(self) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/tensor.py", line 327, in call_method | |
| return wrap_fx_proxy( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/convit.py", line 138, in get_rel_indices | |
| indy = ind.repeat_interleave(img_size, dim=0).repeat_interleave(img_size, dim=1) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train convit_base FAIL | |
| Running timm_models.py convmixer_768_32... | |
| cuda train convmixer_768_32 PASS | |
| Running timm_models.py convnext_base... | |
| cuda train convnext_base PASS | |
| Running timm_models.py crossvit_9_240... | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/crossvit.py", line 281, in scale_image | |
| x = torch.nn.functional.interpolate(x, size=ss, mode='bicubic', align_corners=False) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1091, in run_node | |
| return node.target(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/functional.py", line 3966, in interpolate | |
| return torch._C._nn.upsample_bicubic2d(input, output_size, align_corners, scale_factors) | |
| RuntimeError: Cannot call sizes() on tensor with symbolic sizes/strides | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_function <function interpolate at 0x7f786589a160>(*(FakeTensor(FakeTensor(..., device='meta', size=(s0, s1, s2, s2)), cuda:0),), **{'size': (224, 224), 'mode': 'bicubic', 'align_corners': False}): | |
| Cannot call sizes() on tensor with symbolic sizes/strides | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/crossvit.py", line 418, in forward | |
| xs = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/crossvit.py", line 394, in forward_features | |
| x_ = scale_image(x_, ss, self.crop_scale) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 959, in CALL_FUNCTION_KW | |
| self.call_function(fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/torch.py", line 430, in call_function | |
| tensor_variable = wrap_fx_proxy( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/crossvit.py", line 281, in scale_image | |
| x = torch.nn.functional.interpolate(x, size=ss, mode='bicubic', align_corners=False) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train crossvit_9_240 FAIL | |
| Running timm_models.py cspdarknet53... | |
| cuda train cspdarknet53 PASS | |
| Running timm_models.py deit_base_distilled_patch16_224... | |
| cuda train deit_base_distilled_patch16_224 PASS | |
| Running timm_models.py dla102... | |
| cuda train dla102 PASS | |
| Running timm_models.py dm_nfnet_f0... | |
| ERROR:common:output 0: torch.Size([2, 64, 96, 96]) != torch.Size([2, 256, 48, 48]) | |
| While executing %primals_1 : [#users=2] = placeholder[target=primals_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nfnet.py", line 583, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nfnet.py", line 570, in forward_features | |
| x = self.stem(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nfnet.py", line 574, in <graph break in forward_features> | |
| x = self.stages(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nfnet.py", line 354, in forward | |
| out = self.conv2(self.act2(out)) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/std_conv.py", line 128, in forward | |
| x = pad_same(x, self.kernel_size, self.stride, self.dilation) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/padding.py", line 28, in pad_same | |
| def pad_same(x, k: List[int], s: List[int], d: List[int] = (1, 1), value: float = 0): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([2, 64, 96, 96]) != torch.Size([2, 256, 48, 48]) | |
| While executing %primals_1 : [#users=2] = placeholder[target=primals_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train dm_nfnet_f0 FAIL | |
| Running timm_models.py dpn107... | |
| cuda train dpn107 PASS | |
| Running timm_models.py eca_botnext26ts_256... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1559, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1551, in forward_features | |
| x = self.stages(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1245, in forward | |
| x = self.self_attn(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 137, in forward | |
| _assert(H == self.pos_embed.height, '') | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 152, in <graph break in forward> | |
| attn = (q @ k) * self.scale + self.pos_embed(q) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 68, in forward | |
| def forward(self, q): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train eca_botnext26ts_256 FAIL | |
| Running timm_models.py eca_halonext26ts... | |
| ERROR:common:output 0: torch.Size([64, 8, 8, 16]) != torch.Size([64, 4, 4, 16]) | |
| While executing %primals_1 : [#users=5] = placeholder[target=primals_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1559, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1551, in forward_features | |
| x = self.stages(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1245, in forward | |
| x = self.self_attn(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/halo_attn.py", line 171, in forward | |
| _assert(H % self.block_size == 0, '') | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/halo_attn.py", line 199, in <graph break in forward> | |
| attn = (q @ k.transpose(-1, -2)) * self.scale + self.pos_embed(q) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/halo_attn.py", line 86, in forward | |
| rel_logits_w = rel_logits_1d(q, self.width_rel, permute_mask=(0, 1, 3, 2, 4)) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/halo_attn.py", line 30, in rel_logits_1d | |
| def rel_logits_1d(q, rel_k, permute_mask: List[int]): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([64, 8, 8, 16]) != torch.Size([64, 4, 4, 16]) | |
| While executing %primals_1 : [#users=5] = placeholder[target=primals_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train eca_halonext26ts FAIL | |
| Running timm_models.py ese_vovnet19b_dw... | |
| cuda train ese_vovnet19b_dw PASS | |
| Running timm_models.py fbnetc_100... | |
| cuda train fbnetc_100 PASS | |
| Running timm_models.py fbnetv3_b... | |
| cuda train fbnetv3_b PASS | |
| Running timm_models.py gernet_l... | |
| cuda train gernet_l PASS | |
| Running timm_models.py ghostnet_100... | |
| cuda train ghostnet_100 PASS | |
| Running timm_models.py gluon_inception_v3... | |
| cuda train gluon_inception_v3 PASS | |
| Running timm_models.py gluon_xception65... | |
| cuda train gluon_xception65 PASS | |
| Running timm_models.py gmixer_24_224... | |
| cuda train gmixer_24_224 PASS | |
| Running timm_models.py gmlp_s16_224... | |
| cuda train gmlp_s16_224 PASS | |
| Running timm_models.py hrnet_w18... | |
| ERROR:common: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/hrnet.py", line 713, in stages | |
| yl = self.stage2(xl) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/hrnet.py", line 495, in forward | |
| y = y + fuse_outer[j](x[j]) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1096, in run_node | |
| return nnmodule(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/upsampling.py", line 156, in forward | |
| return F.interpolate(input, self.size, self.scale_factor, self.mode, self.align_corners, | |
| File "/data/users/ezyang/b/pytorch/torch/nn/functional.py", line 3930, in interpolate | |
| return torch._C._nn.upsample_nearest2d(input, output_size, scale_factors) | |
| RuntimeError: Cannot call sizes() on tensor with symbolic sizes/strides | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1057, in get_fake_value | |
| return wrap_fake_exception( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 741, in wrap_fake_exception | |
| return fn() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1058, in <lambda> | |
| lambda: run_node(tx.output, node, args, kwargs, nnmodule) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1100, in run_node | |
| raise RuntimeError( | |
| RuntimeError: Failed running call_module sub1_1_2(*(FakeTensor(FakeTensor(..., device='meta', size=(s0, 18, (s2 - 1)//8 + 1, (s2 - 1)//8 + 1), | |
| grad_fn=<NativeBatchNormBackward0>), cuda:0),), **{}): | |
| Cannot call sizes() on tensor with symbolic sizes/strides | |
| (scroll up for backtrace) | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 286, in catch_errors | |
| return callback(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 476, in _convert_frame | |
| result = inner_convert(frame, cache_size) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 90, in time_wrapper | |
| r = func(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 349, in _convert_frame_assert | |
| return _compile( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 404, in _compile | |
| out_code = transform_code_object(code, transform) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object | |
| transformations(instructions, code_options) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/convert_frame.py", line 392, in transform | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1617, in run | |
| super().run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 910, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 226, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 196, in call_function | |
| return super(UserFunctionVariable, self).call_function(tx, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 67, in call_function | |
| return tx.inline_user_function_return( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 424, in inline_user_function_return | |
| result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1689, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1743, in inline_call_ | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 910, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 226, in call_function | |
| return super().call_function(tx, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 196, in call_function | |
| return super(UserFunctionVariable, self).call_function(tx, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/functions.py", line 67, in call_function | |
| return tx.inline_user_function_return( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 424, in inline_user_function_return | |
| result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1689, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1743, in inline_call_ | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 910, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/nn_module.py", line 183, in call_function | |
| tx.call_function( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/nn_module.py", line 222, in call_function | |
| return tx.inline_user_function_return( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 424, in inline_user_function_return | |
| result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1689, in inline_call | |
| return cls.inline_call_(parent, func, args, kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 1743, in inline_call_ | |
| tracer.run() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 483, in run | |
| and self.step() | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 453, in step | |
| getattr(self, inst.opname)(inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 287, in wrapper | |
| return inner_fn(self, inst) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 910, in CALL_FUNCTION | |
| self.call_function(fn, args, {}) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/nn_module.py", line 183, in call_function | |
| tx.call_function( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/symbolic_convert.py", line 395, in call_function | |
| self.push(fn.call_function(self, args, kwargs)) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/nn_module.py", line 202, in call_function | |
| return wrap_fx_proxy( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 652, in wrap_fx_proxy | |
| return wrap_fx_proxy_cls( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/variables/builder.py", line 693, in wrap_fx_proxy_cls | |
| example_value = get_fake_value(proxy.node, tx) | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/utils.py", line 1070, in get_fake_value | |
| raise TorchRuntimeError() from e | |
| torch._dynamo.exc.TorchRuntimeError: | |
| from user code: | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/hrnet.py", line 713, in stages | |
| yl = self.stage2(xl) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/hrnet.py", line 495, in forward | |
| y = y + fuse_outer[j](x[j]) | |
| Set torch._dynamo.config.verbose=True for more information | |
| You can suppress this exception and fall back to eager by setting: | |
| torch._dynamo.config.suppress_errors = True | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train hrnet_w18 FAIL | |
| Running timm_models.py inception_v3... | |
| cuda train inception_v3 PASS | |
| Running timm_models.py jx_nest_base... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nest.py", line 372, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nest.py", line 359, in forward_features | |
| x = self.patch_embed(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nest.py", line 360, in <graph break in forward_features> | |
| x = self.levels(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/nest.py", line 201, in forward | |
| def forward(self, x): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train jx_nest_base FAIL | |
| Running timm_models.py lcnet_050... | |
| cuda train lcnet_050 PASS | |
| Running timm_models.py levit_128... | |
| WARNING:common:fp64 golden ref were not generated for levit_128 | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 322, in <module> | |
| main(TimmRunnner()) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1651, in main | |
| return maybe_fresh_cache(run, args.cold_start_latency and args.only)( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 810, in inner | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1990, in run | |
| runner.run_one_model( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1336, in run_one_model | |
| status = self.check_accuracy( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1132, in check_accuracy | |
| correct_result = self.run_n_iterations( | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1056, in run_n_iterations | |
| return self.model_iter_fn(mod, inputs, collect_outputs=True) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 312, in forward_and_backward_pass | |
| self.grad_scaler.scale(loss).backward() | |
| File "/data/users/ezyang/b/pytorch/torch/_tensor.py", line 473, in backward | |
| torch.autograd.backward( | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/__init__.py", line 197, in backward | |
| Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass | |
| RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward. | |
| cuda train levit_128 FAIL | |
| Running timm_models.py mixer_b16_224... | |
| cuda train mixer_b16_224 PASS | |
| Running timm_models.py mixnet_l... | |
| cuda train mixnet_l PASS | |
| Running timm_models.py mnasnet_100... | |
| cuda train mnasnet_100 PASS | |
| Running timm_models.py mobilenetv2_100... | |
| cuda train mobilenetv2_100 PASS | |
| Running timm_models.py mobilenetv3_large_100... | |
| cuda train mobilenetv3_large_100 PASS | |
| Running timm_models.py mobilevit_s... | |
| cuda train mobilevit_s PASS | |
| Running timm_models.py nfnet_l0... | |
| cuda train nfnet_l0 PASS | |
| Running timm_models.py pit_b_224... | |
| cuda train pit_b_224 PASS | |
| Running timm_models.py pnasnet5large... | |
| ERROR:common:output 0: torch.Size([96, 1, 5, 5]) != torch.Size([54, 1, 7, 7]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 342, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 318, in forward_features | |
| x_stem_0 = self.cell_stem_0(x_conv_0) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 182, in forward | |
| x_out = self.cell_forward(x_left, x_right) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 121, in cell_forward | |
| x_comb_iter_0_left = self.comb_iter_0_left(x_left) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 122, in <graph break in cell_forward> | |
| x_comb_iter_0_right = self.comb_iter_0_right(x_left) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 125, in <graph break in cell_forward> | |
| x_comb_iter_1_left = self.comb_iter_1_left(x_right) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 70, in forward | |
| x = self.separable_1(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/pnasnet.py", line 49, in forward | |
| x = self.depthwise_conv2d(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 30, in forward | |
| return conv2d_same(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 13, in conv2d_same | |
| def conv2d_same( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([96, 1, 5, 5]) != torch.Size([54, 1, 7, 7]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train pnasnet5large FAIL | |
| Running timm_models.py poolformer_m36... | |
| cuda train poolformer_m36 PASS | |
| Running timm_models.py regnety_002... | |
| cuda train regnety_002 PASS | |
| Running timm_models.py repvgg_a2... | |
| cuda train repvgg_a2 PASS | |
| Running timm_models.py res2net101_26w_4s... | |
| cuda train res2net101_26w_4s PASS | |
| Running timm_models.py res2net50_14w_8s... | |
| cuda train res2net50_14w_8s PASS | |
| Running timm_models.py res2next50... | |
| cuda train res2next50 PASS | |
| Running timm_models.py resmlp_12_224... | |
| cuda train resmlp_12_224 PASS | |
| Running timm_models.py resnest101e... | |
| cuda train resnest101e PASS | |
| Running timm_models.py rexnet_100... | |
| cuda train rexnet_100 PASS | |
| Running timm_models.py sebotnet33ts_256... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1559, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1551, in forward_features | |
| x = self.stages(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/byobnet.py", line 1245, in forward | |
| x = self.self_attn(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 137, in forward | |
| _assert(H == self.pos_embed.height, '') | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 152, in <graph break in forward> | |
| attn = (q @ k) * self.scale + self.pos_embed(q) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/bottleneck_attn.py", line 68, in forward | |
| def forward(self, q): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train sebotnet33ts_256 FAIL | |
| Running timm_models.py selecsls42b... | |
| cuda train selecsls42b PASS | |
| Running timm_models.py spnasnet_100... | |
| cuda train spnasnet_100 PASS | |
| Running timm_models.py swin_base_patch4_window7_224... | |
| cuda train swin_base_patch4_window7_224 PASS | |
| Running timm_models.py swsl_resnext101_32x16d... | |
| cuda train swsl_resnext101_32x16d PASS | |
| Running timm_models.py tf_efficientnet_b0... | |
| ERROR:common:output 0: torch.Size([96, 1, 3, 3]) != torch.Size([144, 1, 5, 5]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 557, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 540, in forward_features | |
| x = self.conv_stem(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 545, in <graph break in forward_features> | |
| x = self.blocks(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet_blocks.py", line 183, in forward | |
| x = self.conv_dw(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 30, in forward | |
| return conv2d_same(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/conv2d_same.py", line 13, in conv2d_same | |
| def conv2d_same( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1065, in compiled_function | |
| outs = CompiledFunction.apply(*no_dupe_args_with_synthetic_bases) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 977, in forward | |
| fw_outs = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 126, in check | |
| assert nv.size() == rv.size(), f"{desc()}: {nv.size()} != {rv.size()}" | |
| AssertionError: output 0: torch.Size([96, 1, 3, 3]) != torch.Size([144, 1, 5, 5]) | |
| While executing %primals_1 : [#users=0] = placeholder[target=primals_1] | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train tf_efficientnet_b0 FAIL | |
| Running timm_models.py tf_mixnet_l... | |
| ERROR:common:Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 308, in <graph break in forward_and_backward_pass> | |
| pred = mod(*cloned_inputs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 557, in forward | |
| x = self.forward_features(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 540, in forward_features | |
| x = self.conv_stem(x) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet.py", line 545, in <graph break in forward_features> | |
| x = self.blocks(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/container.py", line 204, in forward | |
| input = module(input) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/efficientnet_blocks.py", line 181, in forward | |
| x = self.conv_pw(x) | |
| File "/data/users/ezyang/b/pytorch/torch/nn/modules/module.py", line 1427, in _call_impl | |
| return forward_call(*input, **kwargs) | |
| File "/home/ezyang/local/b/pytorch-env/lib/python3.9/site-packages/timm/models/layers/mixed_conv2d.py", line 47, in forward | |
| def forward(self, x): | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1537, in forward | |
| return compiled_function( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1507, in compiled_function | |
| return aot_dispatcher_function(args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1125, in compiled_function | |
| fw_outs_including_aliases.append(input_alias.as_strided(out_tensor_meta.size(), out_tensor_meta.stride(), out_tensor_meta.storage_offset())) | |
| RuntimeError: Expected !is_symbolic() to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train tf_mixnet_l FAIL | |
| Running timm_models.py tinynet_a... | |
| cuda train tinynet_a PASS | |
| Running timm_models.py tnt_s_patch16_224... | |
| cuda train tnt_s_patch16_224 PASS | |
| Running timm_models.py twins_pcpvt_base... | |
| cuda train twins_pcpvt_base FAIL (TIMEOUT) | |
| Running timm_models.py visformer_small... | |
| cuda train visformer_small PASS | |
| Running timm_models.py vit_base_patch16_224... | |
| cuda train vit_base_patch16_224 PASS | |
| Running timm_models.py volo_d1_224... | |
| cuda train volo_d1_224 PASS | |
| Running timm_models.py xcit_large_24_p8_224... | |
| WARNING:common:fp64 golden ref were not generated for xcit_large_24_p8_224 | |
| ERROR:common:output 0: (602112, 1, 21504, 768) != (602112, 784, 28, 1) (mismatch at index 1) | |
| While executing %native_batch_norm_backward : [#users=3] = call_function[target=torch.ops.aten.native_batch_norm_backward.default](args = (%view_1 : Tensor[size=[2, 768, 28, 28], stride=[602112, 1, 21504, 768]], %convolution_2 : Tensor[size=[2, 768, 28, 28], stride=[602112, 784, 28, 1]], %primals_8 : Tensor[size=[768], stride=[1]], %primals_16 : Tensor[size=[768], stride=[1]], %primals_17 : Tensor[size=[768], stride=[1]], %getitem_7 : Tensor[size=[768], stride=[1]], %getitem_8 : Tensor[size=[768], stride=[1]], False, 1e-05, [True, True, True]), kwargs = {}) | |
| Traceback (most recent call last): | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1157, in check_accuracy | |
| new_result = optimized_model_iter_fn( | |
| File "/data/users/ezyang/b/pytorch/torch/_dynamo/eval_frame.py", line 174, in _fn | |
| return fn(*args, **kwargs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/common.py", line 1055, in run_n_iterations | |
| self.model_iter_fn(mod, inputs, collect_outputs=False) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 305, in forward_and_backward_pass | |
| cloned_inputs = clone_inputs(inputs) | |
| File "/data/users/ezyang/b/pytorch/benchmarks/dynamo/timm_models.py", line 312, in <graph break in forward_and_backward_pass> | |
| self.grad_scaler.scale(loss).backward() | |
| File "/data/users/ezyang/b/pytorch/torch/_tensor.py", line 473, in backward | |
| torch.autograd.backward( | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/__init__.py", line 197, in backward | |
| Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass | |
| File "/data/users/ezyang/b/pytorch/torch/autograd/function.py", line 270, in apply | |
| return user_fn(self, *args) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 1037, in backward | |
| out = call_func_with_args( | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 595, in call_func_with_args | |
| out = normalize_as_list(f(args)) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/aot_autograd.py", line 570, in g | |
| return f(*args) | |
| File "/data/users/ezyang/b/pytorch/torch/fx/interpreter.py", line 130, in run | |
| self.env[node] = self.run_node(node) | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 158, in run_node | |
| check(nv, rv, lambda: f"output {i}") | |
| File "/data/users/ezyang/b/pytorch/functorch/_src/compilers.py", line 129, in check | |
| assert same_strides, f"{desc()}: {nv.stride()} != {rv.stride()} (mismatch at index {idx})" | |
| AssertionError: output 0: (602112, 1, 21504, 768) != (602112, 784, 28, 1) (mismatch at index 1) | |
| While executing %native_batch_norm_backward : [#users=3] = call_function[target=torch.ops.aten.native_batch_norm_backward.default](args = (%view_1 : Tensor[size=[2, 768, 28, 28], stride=[602112, 1, 21504, 768]], %convolution_2 : Tensor[size=[2, 768, 28, 28], stride=[602112, 784, 28, 1]], %primals_8 : Tensor[size=[768], stride=[1]], %primals_16 : Tensor[size=[768], stride=[1]], %primals_17 : Tensor[size=[768], stride=[1]], %getitem_7 : Tensor[size=[768], stride=[1]], %getitem_8 : Tensor[size=[768], stride=[1]], False, 1e-05, [True, True, True]), kwargs = {}) | |
| TorchDynamo optimized model failed to run because of following error | |
| cuda train xcit_large_24_p8_224 FAIL |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment