gistfile0.txt

2024-07-04T23:20:39.8000729Z loading model: 0it [00:00, ?it/s]WARNING:common:Model pyhpc_turbulent_kinetic_energy does not support bfloat16, running with amp instead
2024-07-04T23:20:39.9484088Z 
2024-07-04T23:20:39.9485124Z loading model: 0it [00:01, ?it/s]
2024-07-04T23:20:39.9486440Z WARNING:common:Model pyhpc_turbulent_kinetic_energy does not support bfloat16, running with amp instead
2024-07-04T23:20:39.9487446Z cuda eval  pyhpc_turbulent_kinetic_energy     
2024-07-04T23:20:39.9743673Z WARNING:common:Model pyhpc_turbulent_kinetic_energy does not support bfloat16, running with amp instead
2024-07-04T23:21:01.2023259Z ERROR:common:
2024-07-04T23:21:01.2024509Z Traceback (most recent call last):
2024-07-04T23:21:01.2025803Z   File "/var/lib/jenkins/workspace/benchmarks/dynamo/common.py", line 2642, in check_accuracy
2024-07-04T23:21:01.2027561Z     new_result = optimized_model_iter_fn(model_copy, example_inputs)
2024-07-04T23:21:01.2032177Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 435, in _fn
2024-07-04T23:21:01.2033420Z     return fn(*args, **kwargs)
2024-07-04T23:21:01.2034658Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 1121, in __call__
2024-07-04T23:21:01.2035735Z     return self._torchdynamo_orig_callable(
2024-07-04T23:21:01.2036965Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 948, in __call__
2024-07-04T23:21:01.2038000Z     result = self._inner_convert(
2024-07-04T23:21:01.2039151Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 472, in __call__
2024-07-04T23:21:01.2040154Z     return _compile(
2024-07-04T23:21:01.2041243Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_utils_internal.py", line 85, in wrapper_function
2024-07-04T23:21:01.2042494Z     return StrobelightCompileTimeProfiler.profile_compile_time(
2024-07-04T23:21:01.2044057Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_strobelight/compile_time_profiler.py", line 129, in profile_compile_time
2024-07-04T23:21:01.2045293Z     return func(*args, **kwargs)
2024-07-04T23:21:01.2046076Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2046846Z     return func(*args, **kwds)
2024-07-04T23:21:01.2047990Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 817, in _compile
2024-07-04T23:21:01.2049211Z     guarded_code = compile_inner(code, one_graph, hooks, transform)
2024-07-04T23:21:01.2050503Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 233, in time_wrapper
2024-07-04T23:21:01.2051481Z     r = func(*args, **kwargs)
2024-07-04T23:21:01.2052628Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 636, in compile_inner
2024-07-04T23:21:01.2053791Z     out_code = transform_code_object(code, transform)
2024-07-04T23:21:01.2055523Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1270, in transform_code_object
2024-07-04T23:21:01.2056788Z     transformations(instructions, code_options)
2024-07-04T23:21:01.2057956Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 178, in _fn
2024-07-04T23:21:01.2058971Z     return fn(*args, **kwargs)
2024-07-04T23:21:01.2060090Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 582, in transform
2024-07-04T23:21:01.2061103Z     tracer.run()
2024-07-04T23:21:01.2062108Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2476, in run
2024-07-04T23:21:01.2063097Z     super().run()
2024-07-04T23:21:01.2064168Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 904, in run
2024-07-04T23:21:01.2065168Z     while self.step():
2024-07-04T23:21:01.2066208Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 816, in step
2024-07-04T23:21:01.2067311Z     self.dispatch_table[inst.opcode](self, inst)
2024-07-04T23:21:01.2068622Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2667, in RETURN_VALUE
2024-07-04T23:21:01.2069685Z     self._return(inst)
2024-07-04T23:21:01.2071057Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2652, in _return
2024-07-04T23:21:01.2072141Z     self.output.compile_subgraph(
2024-07-04T23:21:01.2073343Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1127, in compile_subgraph
2024-07-04T23:21:01.2074675Z     self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root)
2024-07-04T23:21:01.2075636Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2076627Z     return func(*args, **kwds)
2024-07-04T23:21:01.2077891Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1324, in compile_and_call_fx_graph
2024-07-04T23:21:01.2079077Z     compiled_fn = self.call_user_compiler(gm)
2024-07-04T23:21:01.2080230Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 233, in time_wrapper
2024-07-04T23:21:01.2081217Z     r = func(*args, **kwargs)
2024-07-04T23:21:01.2082395Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1415, in call_user_compiler
2024-07-04T23:21:01.2083683Z     raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
2024-07-04T23:21:01.2085155Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/output_graph.py", line 1396, in call_user_compiler
2024-07-04T23:21:01.2086337Z     compiled_fn = compiler_fn(gm, self.example_inputs())
2024-07-04T23:21:01.2087639Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
2024-07-04T23:21:01.2088791Z     compiled_gm = compiler_fn(gm, example_inputs)
2024-07-04T23:21:01.2090067Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/backends/inductor.py", line 16, in inductor
2024-07-04T23:21:01.2091115Z     return compile_fx(*args, **kwargs)
2024-07-04T23:21:01.2091890Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2092675Z     return func(*args, **kwds)
2024-07-04T23:21:01.2093816Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1550, in compile_fx
2024-07-04T23:21:01.2095010Z     return aot_autograd(
2024-07-04T23:21:01.2096124Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/backends/common.py", line 69, in __call__
2024-07-04T23:21:01.2097307Z     cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
2024-07-04T23:21:01.2098755Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 990, in aot_module_simplified
2024-07-04T23:21:01.2099900Z     compiled_fn = dispatch_and_compile()
2024-07-04T23:21:01.2101179Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 979, in dispatch_and_compile
2024-07-04T23:21:01.2102383Z     compiled_fn, _ = create_aot_dispatcher_function(
2024-07-04T23:21:01.2103599Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 233, in time_wrapper
2024-07-04T23:21:01.2104628Z     r = func(*args, **kwargs)
2024-07-04T23:21:01.2105906Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 708, in create_aot_dispatcher_function
2024-07-04T23:21:01.2107158Z     compiled_fn, fw_metadata = compiler_fn(
2024-07-04T23:21:01.2108658Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line 175, in aot_dispatch_base
2024-07-04T23:21:01.2110072Z     compiled_fw = compiler(fw_module, updated_flat_args)
2024-07-04T23:21:01.2111413Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1260, in fw_compiler_freezing
2024-07-04T23:21:01.2112537Z     optimized_function = inner_compile(
2024-07-04T23:21:01.2113785Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py", line 84, in debug_wrapper
2024-07-04T23:21:01.2115154Z     inner_compiled_fn = compiler_fn(gm, example_inputs)
2024-07-04T23:21:01.2116326Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/debug.py", line 304, in inner
2024-07-04T23:21:01.2117291Z     return fn(*args, **kwargs)
2024-07-04T23:21:01.2118008Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2118783Z     return func(*args, **kwds)
2024-07-04T23:21:01.2119634Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2120414Z     return func(*args, **kwds)
2024-07-04T23:21:01.2121544Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 433, in wrapper
2024-07-04T23:21:01.2122585Z     return fn(*args, **kwargs)
2024-07-04T23:21:01.2123641Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 233, in time_wrapper
2024-07-04T23:21:01.2124683Z     r = func(*args, **kwargs)
2024-07-04T23:21:01.2125836Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 536, in compile_fx_inner
2024-07-04T23:21:01.2126938Z     compiled_graph = FxGraphCache.load(
2024-07-04T23:21:01.2128077Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 1119, in load
2024-07-04T23:21:01.2129215Z     compiled_graph = compile_fx_fn(gm, example_inputs, **fx_kwargs)
2024-07-04T23:21:01.2130197Z   File "/opt/conda/envs/py_3.10/lib/python3.10/contextlib.py", line 79, in inner
2024-07-04T23:21:01.2131000Z     return func(*args, **kwds)
2024-07-04T23:21:01.2132204Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 849, in fx_codegen_and_compile
2024-07-04T23:21:01.2133308Z     graph.run(*example_inputs)
2024-07-04T23:21:01.2134610Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 233, in time_wrapper
2024-07-04T23:21:01.2135643Z     r = func(*args, **kwargs)
2024-07-04T23:21:01.2136660Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 720, in run
2024-07-04T23:21:01.2137597Z     return super().run(*args)
2024-07-04T23:21:01.2138634Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/interpreter.py", line 146, in run
2024-07-04T23:21:01.2139599Z     self.env[node] = self.run_node(node)
2024-07-04T23:21:01.2140727Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 1200, in run_node
2024-07-04T23:21:01.2141702Z     result = super().run_node(n)
2024-07-04T23:21:01.2142775Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/fx/interpreter.py", line 203, in run_node
2024-07-04T23:21:01.2143850Z     return getattr(self, n.op)(n.target, args, kwargs)
2024-07-04T23:21:01.2145116Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 969, in call_function
2024-07-04T23:21:01.2146315Z     raise LoweringException(e, target, args, kwargs).with_traceback(
2024-07-04T23:21:01.2147682Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/graph.py", line 966, in call_function
2024-07-04T23:21:01.2148741Z     out = lowerings[target](*args, **kwargs)
2024-07-04T23:21:01.2149906Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/lowering.py", line 310, in wrapped
2024-07-04T23:21:01.2150901Z     out = decomp_fn(*args, **kwargs)
2024-07-04T23:21:01.2152003Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/lowering.py", line 2317, in copy
2024-07-04T23:21:01.2153003Z     x = to_device(x, self.get_device())
2024-07-04T23:21:01.2154187Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/lowering.py", line 614, in to_device
2024-07-04T23:21:01.2155341Z     return TensorBox.create(ir.DeviceCopy.create(x, device))
2024-07-04T23:21:01.2156726Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/ir.py", line 5301, in create
2024-07-04T23:21:01.2157826Z     and all(r in V.graph.constants for r in x.get_read_names())
2024-07-04T23:21:01.2159107Z   File "/opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/_inductor/ir.py", line 301, in get_read_names
2024-07-04T23:21:01.2160201Z     raise NotImplementedError(f"NYI on {type(self)}")
2024-07-04T23:21:01.2161093Z torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
2024-07-04T23:21:01.2162428Z LoweringException: NotImplementedError: NYI on <class 'torch._inductor.ir.Constant'>
2024-07-04T23:21:01.2163272Z   target: aten.copy.default
2024-07-04T23:21:01.2163723Z   args[0]: TensorBox(StorageBox(
2024-07-04T23:21:01.2164224Z     Pointwise(
2024-07-04T23:21:01.2164601Z       'cuda',
2024-07-04T23:21:01.2164941Z       torch.float64,
2024-07-04T23:21:01.2165310Z       def inner_fn(index):
2024-07-04T23:21:01.2165726Z           i0, i1, i2 = index
2024-07-04T23:21:01.2166210Z           tmp0 = ops.constant(0, torch.float64)
2024-07-04T23:21:01.2166740Z           return tmp0
2024-07-04T23:21:01.2167278Z       ,
2024-07-04T23:21:01.2167595Z       ranges=[204, 204, 26],
2024-07-04T23:21:01.2168041Z       origin_node=full_15,
2024-07-04T23:21:01.2168475Z       origins={full_15}
2024-07-04T23:21:01.2168858Z     )
2024-07-04T23:21:01.2169166Z   ))
2024-07-04T23:21:01.2169828Z   args[1]: Constant(value=0.0, dtype=torch.float64, device=device(type='cpu'))
2024-07-04T23:21:01.2170416Z 
2024-07-04T23:21:01.2170778Z Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
2024-07-04T23:21:01.2171344Z 
2024-07-04T23:21:01.2171350Z 
2024-07-04T23:21:01.2171718Z You can suppress this exception and fall back to eager by setting:
2024-07-04T23:21:01.2172405Z     import torch._dynamo
2024-07-04T23:21:01.2172892Z     torch._dynamo.config.suppress_errors = True
2024-07-04T23:21:01.2173325Z 
2024-07-04T23:21:01.2173695Z TorchDynamo optimized model failed to run because of following error
2024-07-04T23:21:01.2174602Z fail_to_run