Last active
August 31, 2016 19:25
-
-
Save audiodude/a31a489ac4cad20a67319d039abcf497 to your computer and use it in GitHub Desktop.
Stack trace running Magenta on approximately 2700 MIDI files
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| W tensorflow/core/common_runtime/bfc_allocator.cc:270] *****************************************************************************xxxxxxxxxxxxxxxxxxxxxxx | |
| W tensorflow/core/common_runtime/bfc_allocator.cc:271] Ran out of memory trying to allocate 52.51MiB. See logs for memory state. | |
| W tensorflow/core/framework/op_kernel.cc:936] Resource exhausted: OOM when allocating tensor with shape[2830,128,38] | |
| E tensorflow/core/client/tensor_c_api.cc:485] OOM when allocating tensor with shape[2830,128,38] | |
| [[Node: transpose = Transpose[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](padding_fifo_queue_DequeueMany/_60, transpose/perm)]] | |
| [[Node: mul/_68 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_208_mul", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] | |
| INFO:tensorflow:Error reported to Coordinator: <class 'tensorflow.python.framework.errors.ResourceExhaustedError'>, OOM when allocating tensor with shape[2830,128,38] | |
| [[Node: transpose = Transpose[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](padding_fifo_queue_DequeueMany/_60, transpose/perm)]] | |
| [[Node: mul/_68 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_208_mul", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] | |
| Caused by op u'transpose', defined at: | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 49, in <module> | |
| tf.app.run() | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 30, in run | |
| sys.exit(main(sys.argv)) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 45, in main | |
| melody_rnn_train.run(melody_encoder_decoder, basic_rnn_graph.build_graph) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/shared/melody_rnn_train.py", line 206, in run | |
| FLAGS.sequence_example_file) | |
| File "/home/audiodude/magenta/magenta/models/basic_rnn/basic_rnn_graph.py", line 99, in build_graph | |
| swap_memory=True) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 778, in dynamic_rnn | |
| for input_ in flat_input) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 778, in <genexpr> | |
| for input_ in flat_input) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py", line 1051, in transpose | |
| ret = gen_array_ops.transpose(a, perm, name=name) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 2489, in transpose | |
| result = _op_def_lib.apply_op("Transpose", x=x, perm=perm, name=name) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op | |
| op_def=op_def) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2310, in create_op | |
| original_op=self._default_original_op, op_def=op_def) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1232, in __init__ | |
| self._traceback = _extract_stack() | |
| E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled | |
| [[Node: input_producer/input_producer_EnqueueMany = QueueEnqueueMany[Tcomponents=[DT_STRING], _class=["loc:@input_producer"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](input_producer, input_producer/RandomShuffle)]] | |
| E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled | |
| [[Node: padding_fifo_queue_enqueue = QueueEnqueue[Tcomponents=[DT_FLOAT, DT_INT64, DT_INT32], _class=["loc:@padding_fifo_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](padding_fifo_queue, ParseSingleSequenceExample/ParseSingleSequenceExample, ParseSingleSequenceExample/ParseSingleSequenceExample:1, Squeeze/_108)]] | |
| E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled | |
| [[Node: padding_fifo_queue_enqueue = QueueEnqueue[Tcomponents=[DT_FLOAT, DT_INT64, DT_INT32], _class=["loc:@padding_fifo_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](padding_fifo_queue, ParseSingleSequenceExample/ParseSingleSequenceExample, ParseSingleSequenceExample/ParseSingleSequenceExample:1, Squeeze/_108)]] | |
| E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled | |
| [[Node: padding_fifo_queue_enqueue = QueueEnqueue[Tcomponents=[DT_FLOAT, DT_INT64, DT_INT32], _class=["loc:@padding_fifo_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](padding_fifo_queue, ParseSingleSequenceExample/ParseSingleSequenceExample, ParseSingleSequenceExample/ParseSingleSequenceExample:1, Squeeze/_108)]] | |
| E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled | |
| [[Node: padding_fifo_queue_enqueue = QueueEnqueue[Tcomponents=[DT_FLOAT, DT_INT64, DT_INT32], _class=["loc:@padding_fifo_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](padding_fifo_queue, ParseSingleSequenceExample/ParseSingleSequenceExample, ParseSingleSequenceExample/ParseSingleSequenceExample:1, Squeeze/_108)]] | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (256): Total Chunks: 1, Chunks in use: 0 256B allocated for chunks. 4B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (512): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (1024): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (2048): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (4096): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (8192): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (16384): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (32768): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (65536): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (131072): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (262144): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (524288): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (1048576): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (2097152): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (4194304): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (8388608): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (16777216): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (33554432): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (67108864): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (134217728): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:639] Bin (268435456): Total Chunks: 0, Chunks in use: 0 0B allocated for chunks. 0B client-requested for chunks. 0B in use in bin. 0B client-requested in use in bin. | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:656] Bin for 38.07MiB was 32.00MiB, Chunk State: | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00000 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00100 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00500 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00600 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00700 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00800 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a00900 of size 70400 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a11c00 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a12000 of size 7680 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a13e00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a13f00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14000 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14100 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14200 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14300 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14400 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14500 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14600 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14700 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14800 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14900 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14a00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14b00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14c00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14d00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14e00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a14f00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15000 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15100 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15200 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15300 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15400 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15800 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15900 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15a00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15b00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15c00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15d00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15e00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a15f00 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a16300 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a16400 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a16500 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a16600 of size 70400 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a27900 of size 70400 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a38c00 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a39000 of size 7680 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3ae00 of size 1024 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3b200 of size 7680 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3d000 of size 7680 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3ee00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3ef00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a3f000 of size 25600 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a45400 of size 44800 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a50300 of size 70400 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a61600 of size 70400 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a72900 of size 25600 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a78e00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005a78f00 of size 2897920 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:674] Chunk at 0x10005d3c700 of size 72825088 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:683] Free at 0x10005a78d00 of size 256 | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:689] Summary of in-use Chunks by size: | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 40 Chunks of size 256 totalling 10.0KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 6 Chunks of size 1024 totalling 6.0KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 4 Chunks of size 7680 totalling 30.0KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 2 Chunks of size 25600 totalling 50.0KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 1 Chunks of size 44800 totalling 43.8KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 5 Chunks of size 70400 totalling 343.8KiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 1 Chunks of size 2897920 totalling 2.76MiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:692] 1 Chunks of size 72825088 totalling 69.45MiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:696] Sum Total of in-use chunks: 72.69MiB | |
| I tensorflow/core/common_runtime/bfc_allocator.cc:698] Stats: | |
| Limit: 76218368 | |
| InUse: 76218112 | |
| MaxInUse: 76218112 | |
| NumAllocs: 88 | |
| MaxAllocSize: 72825088 | |
| W tensorflow/core/common_runtime/bfc_allocator.cc:270] *********************************************************xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | |
| W tensorflow/core/common_runtime/bfc_allocator.cc:271] Ran out of memory trying to allocate 38.07MiB. See logs for memory state. | |
| W tensorflow/core/framework/op_kernel.cc:936] Resource exhausted: OOM when allocating tensor with shape[2052,128,38] | |
| E tensorflow/core/client/tensor_c_api.cc:485] OOM when allocating tensor with shape[2052,128,38] | |
| [[Node: transpose = Transpose[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](padding_fifo_queue_DequeueMany/_164, transpose/perm)]] | |
| [[Node: RNN/pack/_190 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_853_RNN/pack", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] | |
| Traceback (most recent call last): | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 49, in <module> | |
| tf.app.run() | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 30, in run | |
| sys.exit(main(sys.argv)) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 45, in main | |
| melody_rnn_train.run(melody_encoder_decoder, basic_rnn_graph.build_graph) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/shared/melody_rnn_train.py", line 223, in run | |
| FLAGS.summary_frequency) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/shared/melody_rnn_train.py", line 109, in run_training | |
| tf.logging.info('Training complete.') | |
| File "/usr/lib/python2.7/contextlib.py", line 35, in __exit__ | |
| self.gen.throw(type, value, traceback) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/supervisor.py", line 942, in managed_session | |
| self.stop(close_summary_writer=close_summary_writer) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/supervisor.py", line 768, in stop | |
| stop_grace_period_secs=self._stop_grace_secs) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/coordinator.py", line 357, in join | |
| six.reraise(*self._exc_info_to_raise) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/coordinator.py", line 291, in stop_on_exception | |
| yield | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/coordinator.py", line 451, in run | |
| self.run_loop() | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/supervisor.py", line 972, in run_loop | |
| self._sv.global_step]) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 382, in run | |
| run_metadata_ptr) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 655, in _run | |
| feed_dict_string, options, run_metadata) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 723, in _do_run | |
| target_list, options, run_metadata) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 743, in _do_call | |
| raise type(e)(node_def, op, message) | |
| tensorflow.python.framework.errors.ResourceExhaustedError: OOM when allocating tensor with shape[2830,128,38] | |
| [[Node: transpose = Transpose[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](padding_fifo_queue_DequeueMany/_60, transpose/perm)]] | |
| [[Node: mul/_68 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_208_mul", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] | |
| Caused by op u'transpose', defined at: | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 49, in <module> | |
| tf.app.run() | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 30, in run | |
| sys.exit(main(sys.argv)) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/basic_rnn/basic_rnn_train.py", line 45, in main | |
| melody_rnn_train.run(melody_encoder_decoder, basic_rnn_graph.build_graph) | |
| File "/home/audiodude/magenta/bazel-bin/magenta/models/basic_rnn/basic_rnn_train.runfiles/__main__/magenta/models/shared/melody_rnn_train.py", line 206, in run | |
| FLAGS.sequence_example_file) | |
| File "/home/audiodude/magenta/magenta/models/basic_rnn/basic_rnn_graph.py", line 99, in build_graph | |
| swap_memory=True) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 778, in dynamic_rnn | |
| for input_ in flat_input) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 778, in <genexpr> | |
| for input_ in flat_input) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py", line 1051, in transpose | |
| ret = gen_array_ops.transpose(a, perm, name=name) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 2489, in transpose | |
| result = _op_def_lib.apply_op("Transpose", x=x, perm=perm, name=name) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op | |
| op_def=op_def) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2310, in create_op | |
| original_op=self._default_original_op, op_def=op_def) | |
| File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1232, in __init__ | |
| self._traceback = _extract_stack() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment