Skip to content

Instantly share code, notes, and snippets.

@munyari

munyari/tmp

Created March 16, 2017 02:49
Show Gist options
  • Save munyari/6941118d4f498aa3e1f9a1cdac26bd75 to your computer and use it in GitHub Desktop.
Save munyari/6941118d4f498aa3e1f9a1cdac26bd75 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
tmp
2017-03-15 19:29:40,962 INFO - Preparing to run TaskInstance <TaskInstance: munyari/cats-dogs:48 (id: Bb2AzPHniXPvVkLtXGAkem) (checksum: 608fba09501cdec4356edfc7673992cd) (last update: 2017-03-15 19:29:40.890385) [queued]>
2017-03-15 19:29:40,972 INFO - Starting attempt 1 at 2017-03-15 19:29:40.966058
2017-03-15 19:29:41,243 INFO - Executing command in container: stdbuf -o0 sh command.sh
2017-03-15 19:29:41,243 INFO - Pulling Docker image: floydhub/theano:latest-gpu-py2
2017-03-15 19:29:42,464 INFO - Starting container...
2017-03-15 19:29:42,743 INFO -
################################################################################
2017-03-15 19:29:42,744 INFO - Run Output:
2017-03-15 19:34:18,976 INFO - Using Theano backend.
2017-03-15 19:34:40,710 INFO - Using gpu device 0: Tesla K80 (CNMeM is enabled with initial size: 95.0% of memory, cuDNN 5110)
2017-03-15 19:34:40,712 INFO - /usr/local/lib/python2.7/site-packages/theano/sandbox/cuda/__init__.py:600: UserWarning: Your cuDNN version is more recent than the one Theano officially supports. If you see any problems, try updating Theano or downgrading cuDNN to version 5.
2017-03-15 19:34:40,712 INFO - warnings.warn(warn)
2017-03-15 19:34:56,820 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,599 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,625 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,650 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,676 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,703 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,737 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,769 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,813 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,877 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,940 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:04,999 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:05,061 INFO - (Subtensor{int64}.0, Elemwise{add,no_inplace}.0, Elemwise{add,no_inplace}.0, Subtensor{int64}.0)
2017-03-15 19:35:07,486 INFO - Found 23000 images belonging to 2 classes.
2017-03-15 19:35:07,899 INFO - Found 2000 images belonging to 2 classes.
2017-03-15 19:41:38,532 INFO - 1 #include <Python.h>
2017-03-15 19:41:38,534 INFO - 2 #include <iostream>
2017-03-15 19:41:38,535 INFO - 3 #include "theano_mod_helper.h"
2017-03-15 19:41:38,535 INFO - 4 #include "cuda_ndarray.cuh"
2017-03-15 19:41:38,535 INFO - 5 //////////////////////
2017-03-15 19:41:38,535 INFO - 6 //// Support Code
2017-03-15 19:41:38,536 INFO - 7 //////////////////////
2017-03-15 19:41:38,536 INFO - 8
2017-03-15 19:41:38,536 INFO - 9
2017-03-15 19:41:38,536 INFO - 10 namespace {
2017-03-15 19:41:38,537 INFO - 11 struct __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e {
2017-03-15 19:41:38,537 INFO - 12 PyObject* __ERROR;
2017-03-15 19:41:38,537 INFO - 13
2017-03-15 19:41:38,538 INFO - 14 PyObject* storage_V3;
2017-03-15 19:41:38,538 INFO - 15 PyObject* storage_V1;
2017-03-15 19:41:38,538 INFO - 16
2017-03-15 19:41:38,539 INFO - 17
2017-03-15 19:41:38,539 INFO - 18 __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e() {
2017-03-15 19:41:38,539 INFO - 19 // This is only somewhat safe because we:
2017-03-15 19:41:38,539 INFO - 20 // 1) Are not a virtual class
2017-03-15 19:41:38,540 INFO - 21 // 2) Do not use any virtual classes in the members
2017-03-15 19:41:38,540 INFO - 22 // 3) Deal with mostly POD and pointers
2017-03-15 19:41:38,540 INFO - 23
2017-03-15 19:41:38,540 INFO - 24 // If this changes, we would have to revise this, but for
2017-03-15 19:41:38,541 INFO - 25 // now I am tired of chasing segfaults because
2017-03-15 19:41:38,541 INFO - 26 // initialization code had an error and some pointer has
2017-03-15 19:41:38,541 INFO - 27 // a junk value.
2017-03-15 19:41:38,541 INFO - 28 memset(this, 0, sizeof(*this));
2017-03-15 19:41:38,542 INFO - 29 }
2017-03-15 19:41:38,542 INFO - 30 ~__struct_compiled_op_3a084fd032db206a67ca8121a24aa32e(void) {
2017-03-15 19:41:38,542 INFO - 31 cleanup();
2017-03-15 19:41:38,542 INFO - 32 }
2017-03-15 19:41:38,543 INFO - 33
2017-03-15 19:41:38,543 INFO - 34 int init(PyObject* __ERROR, PyObject* storage_V3, PyObject* storage_V1) {
2017-03-15 19:41:38,543 INFO - 35 Py_XINCREF(storage_V3);
2017-03-15 19:41:38,544 INFO - 36 Py_XINCREF(storage_V1);
2017-03-15 19:41:38,544 INFO - 37 this->storage_V3 = storage_V3;
2017-03-15 19:41:38,544 INFO - 38 this->storage_V1 = storage_V1;
2017-03-15 19:41:38,544 INFO - 39
2017-03-15 19:41:38,544 INFO - 40
2017-03-15 19:41:38,545 INFO - 41
2017-03-15 19:41:38,545 INFO - 42
2017-03-15 19:41:38,545 INFO - 43 this->__ERROR = __ERROR;
2017-03-15 19:41:38,545 INFO - 44 return 0;
2017-03-15 19:41:38,546 INFO - 45 }
2017-03-15 19:41:38,546 INFO - 46 void cleanup(void) {
2017-03-15 19:41:38,546 INFO - 47 __label_1:
2017-03-15 19:41:38,546 INFO - 48
2017-03-15 19:41:38,547 INFO - 49 double __DUMMY_1;
2017-03-15 19:41:38,547 INFO - 50 __label_3:
2017-03-15 19:41:38,547 INFO - 51
2017-03-15 19:41:38,547 INFO - 52 double __DUMMY_3;
2017-03-15 19:41:38,548 INFO - 53 __label_6:
2017-03-15 19:41:38,548 INFO - 54
2017-03-15 19:41:38,548 INFO - 55 double __DUMMY_6;
2017-03-15 19:41:38,549 INFO - 56
2017-03-15 19:41:38,549 INFO - 57 Py_XDECREF(this->storage_V3);
2017-03-15 19:41:38,549 INFO - 58 Py_XDECREF(this->storage_V1);
2017-03-15 19:41:38,549 INFO - 59 }
2017-03-15 19:41:38,550 INFO - 60 int run(void) {
2017-03-15 19:41:38,550 INFO - 61 int __failure = 0;
2017-03-15 19:41:38,550 INFO - 62
2017-03-15 19:41:38,551 INFO - 63 PyObject* py_V1;
2017-03-15 19:41:38,551 INFO - 64 CudaNdarray * V1;
2017-03-15 19:41:38,551 INFO - 65 PyObject* py_V3;
2017-03-15 19:41:38,552 INFO - 66 CudaNdarray * V3;
2017-03-15 19:41:38,552 INFO - 67 {
2017-03-15 19:41:38,552 INFO - 68
2017-03-15 19:41:38,552 INFO - 69 py_V1 = PyList_GET_ITEM(storage_V1, 0);
2017-03-15 19:41:38,553 INFO - 70 {Py_XINCREF(py_V1);}
2017-03-15 19:41:38,553 INFO - 71
2017-03-15 19:41:38,553 INFO - 72 if (py_V1 == Py_None)
2017-03-15 19:41:38,554 INFO - 73 {
2017-03-15 19:41:38,554 INFO - 74 V1 = NULL;
2017-03-15 19:41:38,554 INFO - 75 }
2017-03-15 19:41:38,554 INFO - 76 else
2017-03-15 19:41:38,555 INFO - 77 {
2017-03-15 19:41:38,555 INFO - 78
2017-03-15 19:41:38,555 INFO - 79 assert(py_V1->ob_refcnt >= 2); // There should be at least one ref from the container object,
2017-03-15 19:41:38,555 INFO - 80 // and one ref from the local scope.
2017-03-15 19:41:38,556 INFO - 81
2017-03-15 19:41:38,556 INFO - 82 if (CudaNdarray_Check(py_V1))
2017-03-15 19:41:38,556 INFO - 83 {
2017-03-15 19:41:38,556 INFO - 84 //fprintf(stderr, "c_extract CNDA object w refcnt %p %i\n", py_V1, (py_V1->ob_refcnt));
2017-03-15 19:41:38,557 INFO - 85 V1 = (CudaNdarray*)py_V1;
2017-03-15 19:41:38,557 INFO - 86 //std::cerr << "c_extract " << V1 << '\n';
2017-03-15 19:41:38,557 INFO - 87
2017-03-15 19:41:38,557 INFO - 88
2017-03-15 19:41:38,558 INFO - 89 if (V1->nd != 2)
2017-03-15 19:41:38,558 INFO - 90 {
2017-03-15 19:41:38,558 INFO - 91 PyErr_Format(PyExc_RuntimeError,
2017-03-15 19:41:38,558 INFO - 92 "c_extract: Some CudaNdarray has rank %i, it was supposed to have rank 2",
2017-03-15 19:41:38,559 INFO - 93 V1->nd);
2017-03-15 19:41:38,559 INFO - 94 V1 = NULL;
2017-03-15 19:41:38,559 INFO - 95 {
2017-03-15 19:41:38,560 INFO - 96 __failure = 2;
2017-03-15 19:41:38,560 INFO - 97 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,560 INFO - 98 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,560 INFO - 99 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,561 INFO - 100 "No Python exception was set.");
2017-03-15 19:41:38,561 INFO - 101 }
2017-03-15 19:41:38,561 INFO - 102 goto __label_2;};
2017-03-15 19:41:38,561 INFO - 103 }
2017-03-15 19:41:38,562 INFO - 104 //std::cerr << "c_extract " << V1 << " nd check passed\n";
2017-03-15 19:41:38,562 INFO - 105
2017-03-15 19:41:38,562 INFO - 106
2017-03-15 19:41:38,563 INFO - 107 assert(V1);
2017-03-15 19:41:38,563 INFO - 108 Py_INCREF(py_V1);
2017-03-15 19:41:38,563 INFO - 109 }
2017-03-15 19:41:38,563 INFO - 110 else if (py_V1 == Py_None)
2017-03-15 19:41:38,564 INFO - 111 {
2017-03-15 19:41:38,564 INFO - 112 PyErr_SetString(PyExc_TypeError,
2017-03-15 19:41:38,564 INFO - 113 "expected a CudaNdarray, not None");
2017-03-15 19:41:38,564 INFO - 114 V1 = NULL;
2017-03-15 19:41:38,565 INFO - 115 {
2017-03-15 19:41:38,565 INFO - 116 __failure = 2;
2017-03-15 19:41:38,565 INFO - 117 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,565 INFO - 118 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,566 INFO - 119 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,566 INFO - 120 "No Python exception was set.");
2017-03-15 19:41:38,567 INFO - 121 }
2017-03-15 19:41:38,567 INFO - 122 goto __label_2;};
2017-03-15 19:41:38,567 INFO - 123 }
2017-03-15 19:41:38,567 INFO - 124 else
2017-03-15 19:41:38,567 INFO - 125 {
2017-03-15 19:41:38,568 INFO - 126 //fprintf(stderr, "FAILING c_extract CNDA object w refcnt %p %i\n", py_V1, (py_V1->ob_refcnt));
2017-03-15 19:41:38,568 INFO - 127 PyErr_SetString(PyExc_TypeError, "Argument not a CudaNdarray");
2017-03-15 19:41:38,568 INFO - 128 V1 = NULL;
2017-03-15 19:41:38,568 INFO - 129 {
2017-03-15 19:41:38,569 INFO - 130 __failure = 2;
2017-03-15 19:41:38,569 INFO - 131 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,569 INFO - 132 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,569 INFO - 133 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,570 INFO - 134 "No Python exception was set.");
2017-03-15 19:41:38,570 INFO - 135 }
2017-03-15 19:41:38,570 INFO - 136 goto __label_2;};
2017-03-15 19:41:38,571 INFO - 137 }
2017-03-15 19:41:38,571 INFO - 138 //std::cerr << "c_extract done " << V1 << '\n';
2017-03-15 19:41:38,571 INFO - 139
2017-03-15 19:41:38,571 INFO - 140
2017-03-15 19:41:38,572 INFO - 141 }
2017-03-15 19:41:38,572 INFO - 142
2017-03-15 19:41:38,572 INFO - 143 {
2017-03-15 19:41:38,572 INFO - 144
2017-03-15 19:41:38,573 INFO - 145 py_V3 = PyList_GET_ITEM(storage_V3, 0);
2017-03-15 19:41:38,573 INFO - 146 {Py_XINCREF(py_V3);}
2017-03-15 19:41:38,573 INFO - 147
2017-03-15 19:41:38,573 INFO - 148 assert(py_V3->ob_refcnt >= 2); // There should be at least one ref from the container object,
2017-03-15 19:41:38,574 INFO - 149 // and one ref from the local scope.
2017-03-15 19:41:38,574 INFO - 150
2017-03-15 19:41:38,574 INFO - 151 if (CudaNdarray_Check(py_V3))
2017-03-15 19:41:38,574 INFO - 152 {
2017-03-15 19:41:38,574 INFO - 153 //fprintf(stderr, "c_extract CNDA object w refcnt %p %i\n", py_V3, (py_V3->ob_refcnt));
2017-03-15 19:41:38,575 INFO - 154 V3 = (CudaNdarray*)py_V3;
2017-03-15 19:41:38,575 INFO - 155 //std::cerr << "c_extract " << V3 << '\n';
2017-03-15 19:41:38,575 INFO - 156
2017-03-15 19:41:38,575 INFO - 157
2017-03-15 19:41:38,576 INFO - 158 if (V3->nd != 2)
2017-03-15 19:41:38,576 INFO - 159 {
2017-03-15 19:41:38,576 INFO - 160 PyErr_Format(PyExc_RuntimeError,
2017-03-15 19:41:38,576 INFO - 161 "c_extract: Some CudaNdarray has rank %i, it was supposed to have rank 2",
2017-03-15 19:41:38,577 INFO - 162 V3->nd);
2017-03-15 19:41:38,577 INFO - 163 V3 = NULL;
2017-03-15 19:41:38,577 INFO - 164 {
2017-03-15 19:41:38,577 INFO - 165 __failure = 4;
2017-03-15 19:41:38,578 INFO - 166 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,578 INFO - 167 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,578 INFO - 168 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,578 INFO - 169 "No Python exception was set.");
2017-03-15 19:41:38,578 INFO - 170 }
2017-03-15 19:41:38,579 INFO - 171 goto __label_4;};
2017-03-15 19:41:38,579 INFO - 172 }
2017-03-15 19:41:38,579 INFO - 173 //std::cerr << "c_extract " << V3 << " nd check passed\n";
2017-03-15 19:41:38,579 INFO - 174
2017-03-15 19:41:38,580 INFO - 175
2017-03-15 19:41:38,580 INFO - 176 assert(V3);
2017-03-15 19:41:38,580 INFO - 177 Py_INCREF(py_V3);
2017-03-15 19:41:38,580 INFO - 178 }
2017-03-15 19:41:38,581 INFO - 179 else if (py_V3 == Py_None)
2017-03-15 19:41:38,581 INFO - 180 {
2017-03-15 19:41:38,581 INFO - 181 PyErr_SetString(PyExc_TypeError,
2017-03-15 19:41:38,581 INFO - 182 "expected a CudaNdarray, not None");
2017-03-15 19:41:38,581 INFO - 183 V3 = NULL;
2017-03-15 19:41:38,582 INFO - 184 {
2017-03-15 19:41:38,582 INFO - 185 __failure = 4;
2017-03-15 19:41:38,582 INFO - 186 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,582 INFO - 187 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,582 INFO - 188 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,583 INFO - 189 "No Python exception was set.");
2017-03-15 19:41:38,583 INFO - 190 }
2017-03-15 19:41:38,583 INFO - 191 goto __label_4;};
2017-03-15 19:41:38,583 INFO - 192 }
2017-03-15 19:41:38,583 INFO - 193 else
2017-03-15 19:41:38,584 INFO - 194 {
2017-03-15 19:41:38,584 INFO - 195 //fprintf(stderr, "FAILING c_extract CNDA object w refcnt %p %i\n", py_V3, (py_V3->ob_refcnt));
2017-03-15 19:41:38,584 INFO - 196 PyErr_SetString(PyExc_TypeError, "Argument not a CudaNdarray");
2017-03-15 19:41:38,584 INFO - 197 V3 = NULL;
2017-03-15 19:41:38,584 INFO - 198 {
2017-03-15 19:41:38,584 INFO - 199 __failure = 4;
2017-03-15 19:41:38,585 INFO - 200 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,585 INFO - 201 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,585 INFO - 202 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,585 INFO - 203 "No Python exception was set.");
2017-03-15 19:41:38,585 INFO - 204 }
2017-03-15 19:41:38,585 INFO - 205 goto __label_4;};
2017-03-15 19:41:38,586 INFO - 206 }
2017-03-15 19:41:38,586 INFO - 207 //std::cerr << "c_extract done " << V3 << '\n';
2017-03-15 19:41:38,586 INFO - 208
2017-03-15 19:41:38,586 INFO - 209
2017-03-15 19:41:38,586 INFO - 210 {
2017-03-15 19:41:38,586 INFO - 211 // Op class GpuElemwise
2017-03-15 19:41:38,587 INFO - 212
2017-03-15 19:41:38,587 INFO - 213 //std::cerr << "C_CODE RoundHalfToEven START\n";
2017-03-15 19:41:38,587 INFO - 214 //standard elemwise size checks
2017-03-15 19:41:38,587 INFO - 215
2017-03-15 19:41:38,587 INFO - 216
2017-03-15 19:41:38,587 INFO - 217 int dims[2] = {1,1};
2017-03-15 19:41:38,588 INFO - 218
2017-03-15 19:41:38,588 INFO - 219
2017-03-15 19:41:38,588 INFO - 220 int broadcasts_V3[2] = {0, 0};
2017-03-15 19:41:38,588 INFO - 221
2017-03-15 19:41:38,588 INFO - 222
2017-03-15 19:41:38,588 INFO - 223 //std::cerr << "C_CODE RoundHalfToEven checking input V3\n";
2017-03-15 19:41:38,589 INFO - 224 if (2 != V3->nd)
2017-03-15 19:41:38,589 INFO - 225 {
2017-03-15 19:41:38,589 INFO - 226 PyErr_Format(PyExc_TypeError,
2017-03-15 19:41:38,589 INFO - 227 "need 2 dims, not %i", V3->nd);
2017-03-15 19:41:38,589 INFO - 228 {
2017-03-15 19:41:38,589 INFO - 229 __failure = 5;
2017-03-15 19:41:38,590 INFO - 230 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,590 INFO - 231 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,590 INFO - 232 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,590 INFO - 233 "No Python exception was set.");
2017-03-15 19:41:38,590 INFO - 234 }
2017-03-15 19:41:38,590 INFO - 235 goto __label_5;};
2017-03-15 19:41:38,591 INFO - 236 }
2017-03-15 19:41:38,591 INFO - 237 for (int i = 0; i< 2; ++i)
2017-03-15 19:41:38,591 INFO - 238 {
2017-03-15 19:41:38,591 INFO - 239 dims[i] = (dims[i] == 1) ? CudaNdarray_HOST_DIMS(V3)[i] : dims[i];
2017-03-15 19:41:38,592 INFO - 240 if ((!(broadcasts_V3[i] &&
2017-03-15 19:41:38,592 INFO - 241 CudaNdarray_HOST_DIMS(V3)[i] == 1)) &&
2017-03-15 19:41:38,592 INFO - 242 (dims[i] != CudaNdarray_HOST_DIMS(V3)[i]))
2017-03-15 19:41:38,592 INFO - 243 {
2017-03-15 19:41:38,592 INFO - 244 //std::cerr << "C_CODE RoundHalfToEven checking input V3 failed\n";
2017-03-15 19:41:38,593 INFO - 245 PyErr_Format(PyExc_ValueError,
2017-03-15 19:41:38,593 INFO - 246 "GpuElemwise. Input dimension mis-match. Input"
2017-03-15 19:41:38,593 INFO - 247 " 0 (indices start at 0) has shape[%i] == %i"
2017-03-15 19:41:38,593 INFO - 248 ", but the output's size on that axis is %i.",
2017-03-15 19:41:38,594 INFO - 249 i,
2017-03-15 19:41:38,594 INFO - 250 CudaNdarray_HOST_DIMS(V3)[i],
2017-03-15 19:41:38,594 INFO - 251 dims[i]
2017-03-15 19:41:38,594 INFO - 252 );
2017-03-15 19:41:38,595 INFO - 253 {
2017-03-15 19:41:38,595 INFO - 254 __failure = 5;
2017-03-15 19:41:38,595 INFO - 255 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,595 INFO - 256 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,596 INFO - 257 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,596 INFO - 258 "No Python exception was set.");
2017-03-15 19:41:38,596 INFO - 259 }
2017-03-15 19:41:38,596 INFO - 260 goto __label_5;};
2017-03-15 19:41:38,596 INFO - 261 }
2017-03-15 19:41:38,597 INFO - 262 }
2017-03-15 19:41:38,597 INFO - 263
2017-03-15 19:41:38,597 INFO - 264
2017-03-15 19:41:38,597 INFO - 265 Py_XDECREF(V1);
2017-03-15 19:41:38,598 INFO - 266 V1 = V3;
2017-03-15 19:41:38,598 INFO - 267 Py_INCREF(V1);
2017-03-15 19:41:38,599 INFO - 268 for (int i = 0; (i< 2) && (V1); ++i) {
2017-03-15 19:41:38,599 INFO - 269 if (dims[i] != CudaNdarray_HOST_DIMS(V1)[i])
2017-03-15 19:41:38,599 INFO - 270 {
2017-03-15 19:41:38,600 INFO - 271 PyErr_Format(PyExc_ValueError,
2017-03-15 19:41:38,600 INFO - 272 "GpuElemwise. Output dimension mis-match. Output"
2017-03-15 19:41:38,600 INFO - 273 " 0 (indices start at 0), working inplace"
2017-03-15 19:41:38,601 INFO - 274 " on input 0, has shape[%i] == %i"
2017-03-15 19:41:38,601 INFO - 275 ", but the output's size on that axis is %i.",
2017-03-15 19:41:38,601 INFO - 276 i,
2017-03-15 19:41:38,601 INFO - 277 CudaNdarray_HOST_DIMS(V1)[i],
2017-03-15 19:41:38,602 INFO - 278 dims[i]
2017-03-15 19:41:38,602 INFO - 279 );
2017-03-15 19:41:38,603 INFO - 280 Py_DECREF(V1);
2017-03-15 19:41:38,603 INFO - 281 V1 = NULL;
2017-03-15 19:41:38,603 INFO -
2017-03-15 19:41:38,604 INFO - 282 {
2017-03-15 19:41:38,604 INFO - 283 __failure = 5;
2017-03-15 19:41:38,604 INFO - 284 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,605 INFO - 285 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,605 INFO - 286 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,605 INFO - 287 "No Python exception was set.");
2017-03-15 19:41:38,606 INFO - 288 }
2017-03-15 19:41:38,606 INFO - 289 goto __label_5;};
2017-03-15 19:41:38,606 INFO - 290 }
2017-03-15 19:41:38,607 INFO - 291 }
2017-03-15 19:41:38,607 INFO - 292 //std::cerr << "ELEMWISE NEW V1 nd" << V1->nd << "\n";
2017-03-15 19:41:38,607 INFO - 293 //std::cerr << "ELEMWISE NEW V1 data" << V1->devdata << "\n";
2017-03-15 19:41:38,607 INFO - 294
2017-03-15 19:41:38,608 INFO - 295
2017-03-15 19:41:38,608 INFO - 296 {
2017-03-15 19:41:38,608 INFO - 297 //new block so that failure gotos don't skip over variable initialization
2017-03-15 19:41:38,609 INFO - 298 //std::cerr << "calling callkernel\n";
2017-03-15 19:41:38,609 INFO - 299 if (callkernel_node_3a084fd032db206a67ca8121a24aa32e_0(1, 0, dims
2017-03-15 19:41:38,609 INFO - 300
2017-03-15 19:41:38,610 INFO - 301
2017-03-15 19:41:38,610 INFO - 302 , CudaNdarray_DEV_DATA(V3), CudaNdarray_HOST_STRIDES(V3)
2017-03-15 19:41:38,610 INFO - 303
2017-03-15 19:41:38,610 INFO - 304
2017-03-15 19:41:38,611 INFO - 305 , CudaNdarray_DEV_DATA(V1), CudaNdarray_HOST_STRIDES(V1)
2017-03-15 19:41:38,611 INFO - 306
2017-03-15 19:41:38,611 INFO - 307
2017-03-15 19:41:38,612 INFO - 308 ))
2017-03-15 19:41:38,612 INFO - 309 {
2017-03-15 19:41:38,612 INFO - 310 // error
2017-03-15 19:41:38,612 INFO - 311
2017-03-15 19:41:38,613 INFO - 312
2017-03-15 19:41:38,613 INFO - 313 Py_DECREF(V1);
2017-03-15 19:41:38,613 INFO - 314 V1 = NULL;
2017-03-15 19:41:38,614 INFO - 315
2017-03-15 19:41:38,614 INFO - 316
2017-03-15 19:41:38,614 INFO - 317 {
2017-03-15 19:41:38,615 INFO - 318 __failure = 5;
2017-03-15 19:41:38,615 INFO - 319 if (!PyErr_Occurred()) {
2017-03-15 19:41:38,615 INFO - 320 PyErr_SetString(PyExc_RuntimeError,
2017-03-15 19:41:38,616 INFO - 321 "Unexpected error in an Op's C code. "
2017-03-15 19:41:38,616 INFO - 322 "No Python exception was set.");
2017-03-15 19:41:38,623 INFO - 323 }
2017-03-15 19:41:38,624 INFO - 324 goto __label_5;};
2017-03-15 19:41:38,624 INFO - 325 }
2017-03-15 19:41:38,625 INFO - 326 else // no error
2017-03-15 19:41:38,628 INFO - 327 {
2017-03-15 19:41:38,628 INFO - 328 }
2017-03-15 19:41:38,629 INFO - 329 }
2017-03-15 19:41:38,629 INFO - 330 //std::cerr << "C_CODE RoundHalfToEven END\n";
2017-03-15 19:41:38,629 INFO - 331
2017-03-15 19:41:38,630 INFO - 332 __label_5:
2017-03-15 19:41:38,630 INFO - 333
2017-03-15 19:41:38,631 INFO - 334 double __DUMMY_5;
2017-03-15 19:41:38,631 INFO - 335
2017-03-15 19:41:38,631 INFO - 336 }
2017-03-15 19:41:38,632 INFO - 337 __label_4:
2017-03-15 19:41:38,632 INFO - 338
2017-03-15 19:41:38,632 INFO - 339 //std::cerr << "cleanup " << py_V3 << " " << V3 << "\n";
2017-03-15 19:41:38,633 INFO - 340 //fprintf(stderr, "c_cleanup CNDA py_object w refcnt %p %i\n", py_V3, (py_V3->ob_refcnt));
2017-03-15 19:41:38,633 INFO - 341 if (V3)
2017-03-15 19:41:38,633 INFO - 342 {
2017-03-15 19:41:38,634 INFO - 343 //fprintf(stderr, "c_cleanup CNDA cn_object w refcnt %p %i\n", V3, (V3->ob_refcnt));
2017-03-15 19:41:38,634 INFO - 344 Py_XDECREF(V3);
2017-03-15 19:41:38,634 INFO - 345 }
2017-03-15 19:41:38,635 INFO - 346 //std::cerr << "cleanup done" << py_V3 << "\n";
2017-03-15 19:41:38,635 INFO - 347
2017-03-15 19:41:38,635 INFO - 348 {Py_XDECREF(py_V3);}
2017-03-15 19:41:38,635 INFO - 349
2017-03-15 19:41:38,636 INFO - 350 double __DUMMY_4;
2017-03-15 19:41:38,636 INFO - 351
2017-03-15 19:41:38,636 INFO - 352 }
2017-03-15 19:41:38,637 INFO - 353 __label_2:
2017-03-15 19:41:38,637 INFO - 354
2017-03-15 19:41:38,637 INFO - 355 if (!__failure) {
2017-03-15 19:41:38,638 INFO - 356
2017-03-15 19:41:38,638 INFO - 357 //std::cerr << "sync\n";
2017-03-15 19:41:38,639 INFO - 358 if (NULL == V1) {
2017-03-15 19:41:38,639 INFO - 359 // failure: sync None to storage
2017-03-15 19:41:38,640 INFO - 360 Py_XDECREF(py_V1);
2017-03-15 19:41:38,640 INFO - 361 py_V1 = Py_None;
2017-03-15 19:41:38,640 INFO - 362 Py_INCREF(py_V1);
2017-03-15 19:41:38,643 INFO - 363 }
2017-03-15 19:41:38,644 INFO - 364 else
2017-03-15 19:41:38,644 INFO - 365 {
2017-03-15 19:41:38,645 INFO - 366 if (py_V1 != (PyObject*)V1)
2017-03-15 19:41:38,645 INFO - 367 {
2017-03-15 19:41:38,646 INFO - 368 Py_XDECREF(py_V1);
2017-03-15 19:41:38,646 INFO - 369 py_V1 = (PyObject*)V1;
2017-03-15 19:41:38,646 INFO - 370 Py_INCREF(py_V1);
2017-03-15 19:41:38,647 INFO - 371 }
2017-03-15 19:41:38,647 INFO - 372 assert(py_V1->ob_refcnt);
2017-03-15 19:41:38,648 INFO - 373 }
2017-03-15 19:41:38,648 INFO - 374
2017-03-15 19:41:38,649 INFO - 375 PyObject* old = PyList_GET_ITEM(storage_V1, 0);
2017-03-15 19:41:38,649 INFO - 376 {Py_XINCREF(py_V1);}
2017-03-15 19:41:38,650 INFO - 377 PyList_SET_ITEM(storage_V1, 0, py_V1);
2017-03-15 19:41:38,650 INFO - 378 {Py_XDECREF(old);}
2017-03-15 19:41:38,650 INFO - 379 }
2017-03-15 19:41:38,651 INFO - 380
2017-03-15 19:41:38,651 INFO - 381 //std::cerr << "cleanup " << py_V1 << " " << V1 << "\n";
2017-03-15 19:41:38,651 INFO - 382 //fprintf(stderr, "c_cleanup CNDA py_object w refcnt %p %i\n", py_V1, (py_V1->ob_refcnt));
2017-03-15 19:41:38,652 INFO - 383 if (V1)
2017-03-15 19:41:38,652 INFO - 384 {
2017-03-15 19:41:38,652 INFO - 385 //fprintf(stderr, "c_cleanup CNDA cn_object w refcnt %p %i\n", V1, (V1->ob_refcnt));
2017-03-15 19:41:38,652 INFO - 386 Py_XDECREF(V1);
2017-03-15 19:41:38,653 INFO - 387 }
2017-03-15 19:41:38,653 INFO - 388 //std::cerr << "cleanup done" << py_V1 << "\n";
2017-03-15 19:41:38,653 INFO - 389
2017-03-15 19:41:38,653 INFO - 390 {Py_XDECREF(py_V1);}
2017-03-15 19:41:38,654 INFO - 391
2017-03-15 19:41:38,654 INFO - 392 double __DUMMY_2;
2017-03-15 19:41:38,654 INFO - 393
2017-03-15 19:41:38,654 INFO - 394 }
2017-03-15 19:41:38,655 INFO - 395
2017-03-15 19:41:38,655 INFO - 396
2017-03-15 19:41:38,655 INFO - 397 if (__failure) {
2017-03-15 19:41:38,655 INFO - 398 // When there is a failure, this code puts the exception
2017-03-15 19:41:38,656 INFO - 399 // in __ERROR.
2017-03-15 19:41:38,656 INFO - 400 PyObject* err_type = NULL;
2017-03-15 19:41:38,656 INFO - 401 PyObject* err_msg = NULL;
2017-03-15 19:41:38,657 INFO - 402 PyObject* err_traceback = NULL;
2017-03-15 19:41:38,657 INFO - 403 PyErr_Fetch(&err_type, &err_msg, &err_traceback);
2017-03-15 19:41:38,658 INFO - 404 if (!err_type) {err_type = Py_None;Py_INCREF(Py_None);}
2017-03-15 19:41:38,658 INFO - 405 if (!err_msg) {err_msg = Py_None; Py_INCREF(Py_None);}
2017-03-15 19:41:38,659 INFO - 406 if (!err_traceback) {err_traceback = Py_None; Py_INCREF(Py_None);}
2017-03-15 19:41:38,659 INFO - 407 PyObject* old_err_type = PyList_GET_ITEM(__ERROR, 0);
2017-03-15 19:41:38,660 INFO - 408 PyObject* old_err_msg = PyList_GET_ITEM(__ERROR, 1);
2017-03-15 19:41:38,661 INFO - 409 PyObject* old_err_traceback = PyList_GET_ITEM(__ERROR, 2);
2017-03-15 19:41:38,661 INFO - 410 PyList_SET_ITEM(__ERROR, 0, err_type);
2017-03-15 19:41:38,662 INFO - 411 PyList_SET_ITEM(__ERROR, 1, err_msg);
2017-03-15 19:41:38,662 INFO - 412 PyList_SET_ITEM(__ERROR, 2, err_traceback);
2017-03-15 19:41:38,662 INFO - 413 {Py_XDECREF(old_err_type);}
2017-03-15 19:41:38,663 INFO - 414 {Py_XDECREF(old_err_msg);}
2017-03-15 19:41:38,663 INFO - 415 {Py_XDECREF(old_err_traceback);}
2017-03-15 19:41:38,664 INFO - 416 }
2017-03-15 19:41:38,664 INFO - 417 // The failure code is returned to index what code block failed.
2017-03-15 19:41:38,665 INFO - 418 return __failure;
2017-03-15 19:41:38,665 INFO - 419
2017-03-15 19:41:38,666 INFO - 420 }
2017-03-15 19:41:38,666 INFO - 421 };
2017-03-15 19:41:38,667 INFO - 422 }
2017-03-15 19:41:38,667 INFO - 423
2017-03-15 19:41:38,668 INFO - 424
2017-03-15 19:41:38,669 INFO - 425 static int __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e_executor(__struct_compiled_op_3a084fd032db206a67ca8121a24aa32e* self) {
2017-03-15 19:41:38,669 INFO - 426 return self->run();
2017-03-15 19:41:38,670 INFO - 427 }
2017-03-15 19:41:38,670 INFO - 428
2017-03-15 19:41:38,671 INFO - 429 static void __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e_destructor(void* executor, void* self) {
2017-03-15 19:41:38,671 INFO - 430 delete ((__struct_compiled_op_3a084fd032db206a67ca8121a24aa32e*)self);
2017-03-15 19:41:38,672 INFO - 431 }
2017-03-15 19:41:38,672 INFO - 432
2017-03-15 19:41:38,673 INFO - 433 //////////////////////
2017-03-15 19:41:38,673 INFO - 434 //// Functions
2017-03-15 19:41:38,674 INFO - 435 //////////////////////
2017-03-15 19:41:38,674 INFO - 436 static PyObject * instantiate(PyObject * self, PyObject *argtuple) {
2017-03-15 19:41:38,674 INFO - 437 assert(PyTuple_Check(argtuple));
2017-03-15 19:41:38,675 INFO - 438 if (3 != PyTuple_Size(argtuple)){
2017-03-15 19:41:38,676 INFO - 439 PyErr_Format(PyExc_TypeError, "Wrong number of arguments, expected 3, got %i", (int)PyTuple_Size(argtuple));
2017-03-15 19:41:38,676 INFO - 440 return NULL;
2017-03-15 19:41:38,677 INFO - 441 }
2017-03-15 19:41:38,677 INFO - 442 __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e* struct_ptr = new __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e();
2017-03-15 19:41:38,678 INFO - 443 if (struct_ptr->init( PyTuple_GET_ITEM(argtuple, 0),PyTuple_GET_ITEM(argtuple, 1),PyTuple_GET_ITEM(argtuple, 2) ) != 0) {
2017-03-15 19:41:38,678 INFO - 444 delete struct_ptr;
2017-03-15 19:41:38,679 INFO - 445 return NULL;
2017-03-15 19:41:38,680 INFO - 446 }
2017-03-15 19:41:38,680 INFO - 447 PyObject* thunk = PyCObject_FromVoidPtrAndDesc((void*)(&__struct_compiled_op_3a084fd032db206a67ca8121a24aa32e_executor), struct_ptr, __struct_compiled_op_3a084fd032db206a67ca8121a24aa32e_destructor);
2017-03-15 19:41:38,681 INFO - 448 return thunk; }
2017-03-15 19:41:38,681 INFO - 449
2017-03-15 19:41:38,682 INFO - 450 //////////////////////
2017-03-15 19:41:38,682 INFO - 451 //// Module init
2017-03-15 19:41:38,683 INFO - 452 //////////////////////
2017-03-15 19:41:38,684 INFO - 453 static PyMethodDef MyMethods[] = {
2017-03-15 19:41:38,684 INFO - 454 {"instantiate", instantiate, METH_VARARGS, "undocumented"} ,
2017-03-15 19:41:38,685 INFO - 455 {NULL, NULL, 0, NULL}
2017-03-15 19:41:38,685 INFO - 456 };
2017-03-15 19:41:38,686 INFO - 457 PyMODINIT_FUNC init3a084fd032db206a67ca8121a24aa32e(void){
2017-03-15 19:41:38,687 INFO - 458 (void) Py_InitModule("3a084fd032db206a67ca8121a24aa32e", MyMethods);
2017-03-15 19:41:38,687 INFO - 459 }
2017-03-15 19:41:38,688 INFO - 460
2017-03-15 19:41:38,688 INFO - ===============================
2017-03-15 19:41:38,689 INFO - In file included from /usr/local/include/python2.7/Python.h:8:0,
2017-03-15 19:41:38,690 INFO - from mod.cu:1:
2017-03-15 19:41:38,690 INFO - /usr/local/include/python2.7/pyconfig.h:1190:0: warning: "_POSIX_C_SOURCE" redefined
2017-03-15 19:41:38,691 INFO - #define _POSIX_C_SOURCE 200112L
2017-03-15 19:41:38,691 INFO - ^
2017-03-15 19:41:38,692 INFO - In file included from /usr/local/cuda/include/host_config.h:173:0,
2017-03-15 19:41:38,692 INFO - from /usr/local/cuda/include/cuda_runtime.h:78,
2017-03-15 19:41:38,693 INFO - from <command-line>:0:
2017-03-15 19:41:38,693 INFO - /usr/include/features.h:228:0: note: this is the location of the previous definition
2017-03-15 19:41:38,694 INFO - # define _POSIX_C_SOURCE 200809L
2017-03-15 19:41:38,694 INFO - ^
2017-03-15 19:41:38,695 INFO - In file included from /usr/local/include/python2.7/Python.h:8:0,
2017-03-15 19:41:38,695 INFO - from mod.cu:1:
2017-03-15 19:41:38,696 INFO - /usr/local/include/python2.7/pyconfig.h:1212:0: warning: "_XOPEN_SOURCE" redefined
2017-03-15 19:41:38,696 INFO - #define _XOPEN_SOURCE 600
2017-03-15 19:41:38,696 INFO - ^
2017-03-15 19:41:38,697 INFO - In file included from /usr/local/cuda/include/host_config.h:173:0,
2017-03-15 19:41:38,697 INFO - from /usr/local/cuda/include/cuda_runtime.h:78,
2017-03-15 19:41:38,698 INFO - from <command-line>:0:
2017-03-15 19:41:38,698 INFO - /usr/include/features.h:169:0: note: this is the location of the previous definition
2017-03-15 19:41:38,698 INFO - # define _XOPEN_SOURCE 700
2017-03-15 19:41:38,699 INFO - ^
2017-03-15 19:41:38,699 INFO - mod.cu(299): error: identifier "callkernel_node_3a084fd032db206a67ca8121a24aa32e_0" is undefined
2017-03-15 19:41:38,700 INFO - 1 error detected in the compilation of "/tmp/tmpxft_00000ac6_00000000-9_mod.cpp1.ii".
2017-03-15 19:41:38,700 INFO - ['nvcc', '-shared', '-O3', '--maxrregcount=32', '-use_fast_math', '-arch=sm_37', '-m64', '-Xcompiler', '-fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=c72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden', '-Xlinker', '-rpath,/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray', '-I/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray', '-I/usr/local/cuda/include', '-I/usr/local/lib/python2.7/site-packages/numpy/core/include', '-I/usr/local/include/python2.7', '-I/usr/local/lib/python2.7/site-packages/theano/gof', '-I/usr/local/lib/python2.7/site-packages/theano/sandbox/cuda', '-o', '/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/tmp1PLv0s/3a084fd032db206a67ca8121a24aa32e.so', 'mod.cu', '-L/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray', '-L/usr/local/lib', '-lcudart', '-lcublas', '-lcuda_ndarray', '-lpython2.7']
2017-03-15 19:41:38,700 INFO - Traceback (most recent call last):
2017-03-15 19:41:38,701 INFO - File "demo1.py", line 129, in <module>
2017-03-15 19:41:38,701 INFO - nb_epoch=20, validation_data=valid_batches, nb_val_samples=valid_batches.nb_sample)
2017-03-15 19:41:38,702 INFO - File "/usr/local/lib/python2.7/site-packages/keras/models.py", line 935, in fit_generator
2017-03-15 19:41:38,702 INFO - initial_epoch=initial_epoch)
2017-03-15 19:41:38,702 INFO - File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 1454, in fit_generator
2017-03-15 19:41:38,703 INFO - self._make_train_function()
2017-03-15 19:41:38,703 INFO - File "/usr/local/lib/python2.7/site-packages/keras/engine/training.py", line 767, in _make_train_function
2017-03-15 19:41:38,703 INFO - **self._function_kwargs)
2017-03-15 19:41:38,704 INFO - File "/usr/local/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 969, in function
2017-03-15 19:41:38,704 INFO - return Function(inputs, outputs, updates=updates, **kwargs)
2017-03-15 19:41:38,705 INFO - File "/usr/local/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 955, in __init__
2017-03-15 19:41:38,705 INFO - **kwargs)
2017-03-15 19:41:38,705 INFO - File "/usr/local/lib/python2.7/site-packages/theano/compile/function.py", line 320, in function
2017-03-15 19:41:38,706 INFO - output_keys=output_keys)
2017-03-15 19:41:38,706 INFO - File "/usr/local/lib/python2.7/site-packages/theano/compile/pfunc.py", line 479, in pfunc
2017-03-15 19:41:38,707 INFO - output_keys=output_keys)
2017-03-15 19:41:38,707 INFO - File "/usr/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 1777, in orig_function
2017-03-15 19:41:38,707 INFO - defaults)
2017-03-15 19:41:38,708 INFO - File "/usr/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 1641, in create
2017-03-15 19:41:38,708 INFO - input_storage=input_storage_lists, storage_map=storage_map)
2017-03-15 19:41:38,708 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/link.py", line 690, in make_thunk
2017-03-15 19:41:38,709 INFO - storage_map=storage_map)[:3]
2017-03-15 19:41:38,709 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/vm.py", line 1003, in make_all
2017-03-15 19:41:38,710 INFO - no_recycling))
2017-03-15 19:41:38,710 INFO - File "/usr/local/lib/python2.7/site-packages/theano/sandbox/cuda/__init__.py", line 256, in make_thunk
2017-03-15 19:41:38,710 INFO - compute_map, no_recycling)
2017-03-15 19:41:38,711 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/op.py", line 970, in make_thunk
2017-03-15 19:41:38,711 INFO - no_recycling)
2017-03-15 19:41:38,711 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/op.py", line 879, in make_c_thunk
2017-03-15 19:41:38,712 INFO - output_storage=node_output_storage)
2017-03-15 19:41:38,712 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/cc.py", line 1200, in make_thunk
2017-03-15 19:41:38,713 INFO - keep_lock=keep_lock)
2017-03-15 19:41:38,713 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/cc.py", line 1143, in __compile__
2017-03-15 19:41:38,713 INFO - keep_lock=keep_lock)
2017-03-15 19:41:38,714 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/cc.py", line 1595, in cthunk_factory
2017-03-15 19:41:38,714 INFO - key=key, lnk=self, keep_lock=keep_lock)
2017-03-15 19:41:38,714 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/cmodule.py", line 1142, in module_from_key
2017-03-15 19:41:38,715 INFO - module = lnk.compile_cmodule(location)
2017-03-15 19:41:38,715 INFO - File "/usr/local/lib/python2.7/site-packages/theano/gof/cc.py", line 1506, in compile_cmodule
2017-03-15 19:41:38,716 INFO - preargs=preargs)
2017-03-15 19:41:38,716 INFO - File "/usr/local/lib/python2.7/site-packages/theano/sandbox/cuda/nvcc_compiler.py", line 399, in compile_str
2017-03-15 19:41:38,716 INFO - 'for cmd', ' '.join(cmd))
2017-03-15 19:41:38,717 INFO - Exception: ('The following error happened while compiling the node', GpuElemwise{RoundHalfToEven}[(0, 0)](GpuElemwise{Composite{scalar_sigmoid((i0 + i1))},no_inplace}.0), '\n', 'nvcc return status', 2, 'for cmd', 'nvcc -shared -O3 --maxrregcount=32 -use_fast_math -arch=sm_37 -m64 -Xcompiler -fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=c72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden -Xlinker -rpath,/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray -I/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray -I/usr/local/cuda/include -I/usr/local/lib/python2.7/site-packages/numpy/core/include -I/usr/local/include/python2.7 -I/usr/local/lib/python2.7/site-packages/theano/gof -I/usr/local/lib/python2.7/site-packages/theano/sandbox/cuda -o /root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/tmp1PLv0s/3a084fd032db206a67ca8121a24aa32e.so mod.cu -L/root/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/cuda_ndarray -L/usr/local/lib -lcudart -lcublas -lcuda_ndarray -lpython2.7', '[GpuElemwise{RoundHalfToEven}[(0, 0)](<CudaNdarrayType(float32, matrix)>)]')
2017-03-15 19:41:39,107 INFO -
################################################################################
2017-03-15 19:41:39,108 INFO - Waiting for container to complete...
2017-03-15 19:41:39,536 ERROR - [failed] Execution failed in 718 seconds for TaskInstance <TaskInstance: munyari/cats-dogs:48 (id: Bb2AzPHniXPvVkLtXGAkem) (checksum: 608fba09501cdec4356edfc7673992cd) (last update: 2017-03-15 19:41:39.529960) [failed]>
2017-03-15 19:41:39,537 ERROR - Docker container id af5e2f54c99363587d3ce6ad0b3280096e52acb1d440e342a68b73d6352c8809 failed
Traceback (most recent call last):
File "/app/floyd/models/task_instance.py", line 614, in run
self.execute(module, context)
File "/app/floyd/models/task_instance.py", line 795, in execute
self.container.execute(module, context)
File "/app/floyd/containers/docker_container.py", line 500, in execute
self.wait()
File "/app/floyd/containers/docker_container.py", line 372, in wait
raise FloydException("Docker container id {} failed".format(self.docker_uuid))
floyd.exceptions.FloydException: Docker container id af5e2f54c99363587d3ce6ad0b3280096e52acb1d440e342a68b73d6352c8809 failed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment