Skip to content

Instantly share code, notes, and snippets.

/gist:5548936

Created May 9, 2013 17:11
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/5548936 to your computer and use it in GitHub Desktop.
Save anonymous/5548936 to your computer and use it in GitHub Desktop.
Download ZIP
output of python -c "import theano".
Raw
gistfile1.txt
00001 #include <Python.h>
00002 #include "structmember.h"
00003 #include <sys/time.h>
00004
00005 // Old Python compatibility from here:
00006 // http://www.python.org/dev/peps/pep-0353/
00007 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
00008 typedef int Py_ssize_t;
00009 #define PY_SSIZE_T_MAX INT_MAX
00010 #define PY_SSIZE_T_MIN INT_MIN
00011 // This one was taken from:
00012 // http://svn.python.org/projects/python/trunk/Modules/_ctypes/ctypes.h
00013 #define PyNumber_AsSsize_t(ob, exc) PyInt_AsLong(ob)
00014 #endif
00015
00016 /**
00017
00018 TODO:
00019 - Check max supported depth of recursion
00020 - CLazyLinker should add context information to errors caught during evaluation. Say what node we were on, add the traceback attached to the node.
00021 - Clear containers of fully-useed intermediate results if allow_gc is 1
00022 - Add timers for profiling
00023 - Add support for profiling space used.
00024
00025
00026 */
00027 static double pytime(const struct timeval * tv)
00028 {
00029 struct timeval t;
00030 if (!tv)
00031 {
00032 tv = &t;
00033 gettimeofday(&t, NULL);
00034 }
00035 return (double) tv->tv_sec + (double) tv->tv_usec / 1000000.0;
00036 }
00037
00038 /**
00039 Helper routine to convert a PyList of integers to a c array of integers.
00040 */
00041 static int unpack_list_of_ssize_t(PyObject * pylist, Py_ssize_t **dst, Py_ssize_t *len,
00042 const char* kwname)
00043 {
00044 Py_ssize_t buflen, *buf;
00045 if (!PyList_Check(pylist))
00046 {
00047 PyErr_Format(PyExc_TypeError, "%s must be list", kwname);
00048 return -1;
00049 }
00050 assert (NULL == *dst);
00051 *len = buflen = PyList_Size(pylist);
00052 *dst = buf = (Py_ssize_t*)calloc(buflen, sizeof(Py_ssize_t));
00053 assert(buf);
00054 for (int ii = 0; ii < buflen; ++ii)
00055 {
00056 PyObject * el_i = PyList_GetItem(pylist, ii);
00057 Py_ssize_t n_i = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00058 if (PyErr_Occurred())
00059 {
00060 free(buf);
00061 *dst = NULL;
00062 return -1;
00063 }
00064 buf[ii] = n_i;
00065 }
00066 return 0;
00067 }
00068
00069 /**
00070
00071 CLazyLinker
00072
00073
00074 */
00075 typedef struct {
00076 PyObject_HEAD
00077 /* Type-specific fields go here. */
00078 PyObject * nodes; // the python list of nodes
00079 PyObject * thunks; // python list of thunks
00080 PyObject * pre_call_clear; //list of cells to clear on call.
00081 int allow_gc;
00082 Py_ssize_t n_applies;
00083 int n_vars; // number of variables in the graph
00084 int * var_computed; // 1 or 0 for every variable
00085 PyObject ** var_computed_cells;
00086 PyObject ** var_value_cells;
00087 Py_ssize_t **dependencies; // list of vars dependencies for GC
00088 Py_ssize_t *n_dependencies;
00089
00090 Py_ssize_t n_output_vars;
00091 Py_ssize_t * output_vars; // variables that *must* be evaluated by call
00092
00093 int * is_lazy; // 1 or 0 for every thunk
00094
00095 Py_ssize_t * var_owner; // nodes[[var_owner[var_idx]]] is var[var_idx]->owner
00096 int * var_has_owner; // 1 or 0
00097
00098 Py_ssize_t * node_n_inputs;
00099 Py_ssize_t * node_n_outputs;
00100 Py_ssize_t ** node_inputs;
00101 Py_ssize_t ** node_outputs;
00102 Py_ssize_t * node_inputs_outputs_base; // node_inputs and node_outputs point into this
00103 Py_ssize_t * node_n_prereqs;
00104 Py_ssize_t ** node_prereqs;
00105
00106 Py_ssize_t * update_storage; // input cells to update with the last outputs in output_vars
00107 Py_ssize_t n_updates;
00108
00109 void ** thunk_cptr_fn;
00110 void ** thunk_cptr_data;
00111 PyObject * call_times;
00112 PyObject * call_counts;
00113 int do_timing;
00114 int need_update_inputs;
00115 int position_of_error; // -1 for no error, otw the index into `thunks` that failed.
00116 } CLazyLinker;
00117
00118
00119 static void
00120 CLazyLinker_dealloc(PyObject* _self)
00121 {
00122 CLazyLinker* self = (CLazyLinker *) _self;
00123 free(self->thunk_cptr_fn);
00124 free(self->thunk_cptr_data);
00125
00126 free(self->is_lazy);
00127
00128 free(self->update_storage);
00129
00130 if (self->node_n_prereqs)
00131 {
00132 for (int i = 0; i < self->n_applies; ++i)
00133 {
00134 free(self->node_prereqs[i]);
00135 }
00136 }
00137 free(self->node_n_prereqs);
00138 free(self->node_prereqs);
00139 free(self->node_inputs_outputs_base);
00140 free(self->node_n_inputs);
00141 free(self->node_n_outputs);
00142 free(self->node_inputs);
00143 free(self->node_outputs);
00144
00145 if (self->dependencies)
00146 {
00147 for (int i = 0; i < self->n_vars; ++i)
00148 {
00149 free(self->dependencies[i]);
00150 }
00151 free(self->dependencies);
00152 free(self->n_dependencies);
00153 }
00154
00155 free(self->var_owner);
00156 free(self->var_has_owner);
00157 free(self->var_computed);
00158 if (self->var_computed_cells)
00159 {
00160 for (int i = 0; i < self->n_vars; ++i)
00161 {
00162 Py_DECREF(self->var_computed_cells[i]);
00163 Py_DECREF(self->var_value_cells[i]);
00164 }
00165 }
00166 free(self->var_computed_cells);
00167 free(self->var_value_cells);
00168 free(self->output_vars);
00169
00170 Py_XDECREF(self->nodes);
00171 Py_XDECREF(self->thunks);
00172 Py_XDECREF(self->call_times);
00173 Py_XDECREF(self->call_counts);
00174 Py_XDECREF(self->pre_call_clear);
00175 self->ob_type->tp_free((PyObject*)self);
00176 }
00177 static PyObject *
00178 CLazyLinker_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
00179 {
00180 CLazyLinker *self;
00181
00182 self = (CLazyLinker *)type->tp_alloc(type, 0);
00183 if (self != NULL) {
00184 self->nodes = NULL;
00185 self->thunks = NULL;
00186 self->pre_call_clear = NULL;
00187
00188 self->allow_gc = 1;
00189 self->n_applies = 0;
00190 self->n_vars = 0;
00191 self->var_computed = NULL;
00192 self->var_computed_cells = NULL;
00193 self->var_value_cells = NULL;
00194 self->dependencies = NULL;
00195 self->n_dependencies = NULL;
00196
00197 self->n_output_vars = 0;
00198 self->output_vars = NULL;
00199
00200 self->is_lazy = NULL;
00201
00202 self->var_owner = NULL;
00203 self->var_has_owner = NULL;
00204
00205 self->node_n_inputs = NULL;
00206 self->node_n_outputs = NULL;
00207 self->node_inputs = NULL;
00208 self->node_outputs = NULL;
00209 self->node_inputs_outputs_base = NULL;
00210 self->node_prereqs = NULL;
00211 self->node_n_prereqs = NULL;
00212
00213 self->update_storage = NULL;
00214 self->n_updates = 0;
00215
00216 self->thunk_cptr_data = NULL;
00217 self->thunk_cptr_fn = NULL;
00218 self->call_times = NULL;
00219 self->call_counts = NULL;
00220 self->do_timing = 0;
00221
00222 self->need_update_inputs = 0;
00223 self->position_of_error = -1;
00224 }
00225 return (PyObject *)self;
00226 }
00227
00228 static int
00229 CLazyLinker_init(CLazyLinker *self, PyObject *args, PyObject *kwds)
00230 {
00231 static char *kwlist[] = {
00232 (char*)"nodes",
00233 (char*)"thunks",
00234 (char*)"pre_call_clear",
00235 (char*)"allow_gc",
00236 (char*)"call_counts",
00237 (char*)"call_times",
00238 (char*)"compute_map_list",
00239 (char*)"storage_map_list",
00240 (char*)"base_input_output_list",
00241 (char*)"node_n_inputs",
00242 (char*)"node_n_outputs",
00243 (char*)"node_input_offset",
00244 (char*)"node_output_offset",
00245 (char*)"var_owner",
00246 (char*)"is_lazy_list",
00247 (char*)"output_vars",
00248 (char*)"node_prereqs",
00249 (char*)"node_output_size",
00250 (char*)"update_storage",
00251 (char*)"dependencies",
00252 NULL};
00253
00254 PyObject *compute_map_list=NULL,
00255 *storage_map_list=NULL,
00256 *base_input_output_list=NULL,
00257 *node_n_inputs=NULL,
00258 *node_n_outputs=NULL,
00259 *node_input_offset=NULL,
00260 *node_output_offset=NULL,
00261 *var_owner=NULL,
00262 *is_lazy=NULL,
00263 *output_vars=NULL,
00264 *node_prereqs=NULL,
00265 *node_output_size=NULL,
00266 *update_storage=NULL,
00267 *dependencies=NULL;
00268
00269 assert(!self->nodes);
00270 if (! PyArg_ParseTupleAndKeywords(args, kwds, "OOOiOOOOOOOOOOOOOOOO", kwlist,
00271 &self->nodes,
00272 &self->thunks,
00273 &self->pre_call_clear,
00274 &self->allow_gc,
00275 &self->call_counts,
00276 &self->call_times,
00277 &compute_map_list,
00278 &storage_map_list,
00279 &base_input_output_list,
00280 &node_n_inputs,
00281 &node_n_outputs,
00282 &node_input_offset,
00283 &node_output_offset,
00284 &var_owner,
00285 &is_lazy,
00286 &output_vars,
00287 &node_prereqs,
00288 &node_output_size,
00289 &update_storage,
00290 &dependencies
00291 ))
00292 return -1;
00293 Py_INCREF(self->nodes);
00294 Py_INCREF(self->thunks);
00295 Py_INCREF(self->pre_call_clear);
00296 Py_INCREF(self->call_counts);
00297 Py_INCREF(self->call_times);
00298
00299 Py_ssize_t n_applies = PyList_Size(self->nodes);
00300
00301 self->n_applies = n_applies;
00302 self->n_vars = PyList_Size(var_owner);
00303
00304 if (PyList_Size(self->thunks) != n_applies) return -1;
00305 if (PyList_Size(self->call_counts) != n_applies) return -1;
00306 if (PyList_Size(self->call_times) != n_applies) return -1;
00307
00308 // allocated and initialize thunk_cptr_data and thunk_cptr_fn
00309 if (n_applies)
00310 {
00311 self->thunk_cptr_data = (void**)calloc(n_applies, sizeof(void*));
00312 self->thunk_cptr_fn = (void**)calloc(n_applies, sizeof(void*));
00313 self->is_lazy = (int*)calloc(n_applies, sizeof(int));
00314 self->node_prereqs = (Py_ssize_t**)calloc(n_applies, sizeof(Py_ssize_t*));
00315 self->node_n_prereqs = (Py_ssize_t*)calloc(n_applies, sizeof(Py_ssize_t));
00316 assert(self->node_prereqs);
00317 assert(self->node_n_prereqs);
00318 assert(self->is_lazy);
00319 assert(self->thunk_cptr_fn);
00320 assert(self->thunk_cptr_data);
00321
00322 for (int i = 0; i < n_applies; ++i)
00323 {
00324 PyObject * thunk = PyList_GetItem(self->thunks, i);
00325 //thunk is borrowed
00326 if (PyObject_HasAttrString(thunk, "cthunk"))
00327 {
00328 PyObject * cthunk = PyObject_GetAttrString(thunk, "cthunk");
00329 //new reference
00330 assert (cthunk && PyCObject_Check(cthunk));
00331 self->thunk_cptr_fn[i] = PyCObject_AsVoidPtr(cthunk);
00332 self->thunk_cptr_data[i] = PyCObject_GetDesc(cthunk);
00333 Py_DECREF(cthunk);
00334 // cthunk is kept alive by membership in self->thunks
00335 }
00336
00337 PyObject * el_i = PyList_GetItem(is_lazy, i);
00338 self->is_lazy[i] = PyNumber_AsSsize_t(el_i, NULL);
00339
00340 /* now get the prereqs */
00341 el_i = PyList_GetItem(node_prereqs, i);
00342 assert (PyList_Check(el_i));
00343 self->node_n_prereqs[i] = PyList_Size(el_i);
00344 if (self->node_n_prereqs[i])
00345 {
00346 self->node_prereqs[i] = (Py_ssize_t*)malloc(
00347 PyList_Size(el_i)*sizeof(Py_ssize_t));
00348 for (int j = 0; j < PyList_Size(el_i); ++j)
00349 {
00350 PyObject * el_ij = PyList_GetItem(el_i, j);
00351 Py_ssize_t N = PyNumber_AsSsize_t(el_ij, PyExc_IndexError);
00352 if (PyErr_Occurred())
00353 return -1;
00354 // N < n. variables
00355 assert(N < PyList_Size(var_owner));
00356 self->node_prereqs[i][j] = N;
00357 }
00358 }
00359 }
00360 }
00361 if (PyList_Check(base_input_output_list))
00362 {
00363 Py_ssize_t n_inputs_outputs_base = PyList_Size(base_input_output_list);
00364 self->node_inputs_outputs_base = (Py_ssize_t*)calloc(n_inputs_outputs_base,sizeof(Py_ssize_t));
00365 assert(self->node_inputs_outputs_base);
00366 for (int i = 0; i < n_inputs_outputs_base; ++i)
00367 {
00368 PyObject *el_i = PyList_GetItem(base_input_output_list, i);
00369 Py_ssize_t idx = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00370 if (PyErr_Occurred()) return -1;
00371 self->node_inputs_outputs_base[i] = idx;
00372 }
00373 self->node_n_inputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
00374 assert(self->node_n_inputs);
00375 self->node_n_outputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
00376 assert(self->node_n_outputs);
00377 self->node_inputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
00378 assert(self->node_inputs);
00379 self->node_outputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
00380 assert(self->node_outputs);
00381 for (int i = 0; i < n_applies; ++i)
00382 {
00383 Py_ssize_t N;
00384 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_inputs, i),PyExc_IndexError);
00385 if (PyErr_Occurred()) return -1;
00386 assert (N <= n_inputs_outputs_base);
00387 self->node_n_inputs[i] = N;
00388 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_outputs, i),PyExc_IndexError);
00389 if (PyErr_Occurred()) return -1;
00390 assert (N <= n_inputs_outputs_base);
00391 self->node_n_outputs[i] = N;
00392 N = PyNumber_AsSsize_t(PyList_GetItem(node_input_offset, i),PyExc_IndexError);
00393 if (PyErr_Occurred()) return -1;
00394 assert (N <= n_inputs_outputs_base);
00395 self->node_inputs[i] = &self->node_inputs_outputs_base[N];
00396 N = PyNumber_AsSsize_t(PyList_GetItem(node_output_offset, i),PyExc_IndexError);
00397 if (PyErr_Occurred()) return -1;
00398 assert (N <= n_inputs_outputs_base);
00399 self->node_outputs[i] = &self->node_inputs_outputs_base[N];
00400 }
00401 }
00402 else
00403 {
00404 PyErr_SetString(PyExc_TypeError, "base_input_output_list must be list");
00405 return -1;
00406 }
00407
00408 // allocation for var_owner
00409 if (PyList_Check(var_owner))
00410 {
00411 self->var_owner = (Py_ssize_t*)calloc(self->n_vars,sizeof(Py_ssize_t));
00412 self->var_has_owner = (int*)calloc(self->n_vars,sizeof(int));
00413 self->var_computed = (int*)calloc(self->n_vars,sizeof(int));
00414 self->var_computed_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
00415 self->var_value_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
00416 for (int i = 0; i < self->n_vars; ++i)
00417 {
00418 PyObject * el_i = PyList_GetItem(var_owner, i);
00419 if (el_i == Py_None)
00420 {
00421 self->var_has_owner[i] = 0;
00422 }
00423 else
00424 {
00425 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00426 if (PyErr_Occurred()) return -1;
00427 assert (N <= n_applies);
00428 self->var_owner[i] = N;
00429 self->var_has_owner[i] = 1;
00430 }
00431 self->var_computed_cells[i] = PyList_GetItem(compute_map_list, i);
00432 Py_INCREF(self->var_computed_cells[i]);
00433 self->var_value_cells[i] = PyList_GetItem(storage_map_list, i);
00434 Py_INCREF(self->var_value_cells[i]);
00435 }
00436 }
00437 else
00438 {
00439 PyErr_SetString(PyExc_TypeError, "var_owner must be list");
00440 return -1;
00441 }
00442
00443 if (dependencies != Py_None)
00444 {
00445 self->dependencies = (Py_ssize_t**)calloc(self->n_vars, sizeof(Py_ssize_t *));
00446 self->n_dependencies = (Py_ssize_t*)calloc(self->n_vars, sizeof(Py_ssize_t));
00447 assert(self->dependencies);
00448 assert(self->n_dependencies);
00449
00450 for (int i = 0; i < self->n_vars; ++i)
00451 {
00452 PyObject *tmp = PyList_GetItem(dependencies, i);
00453 // refcounting - tmp is borrowed
00454 if (unpack_list_of_ssize_t(tmp, &self->dependencies[i], &self->n_dependencies[i],
00455 "dependencies"))
00456 return -1;
00457 }
00458 }
00459
00460 if (unpack_list_of_ssize_t(output_vars, &self->output_vars, &self->n_output_vars,
00461 "output_vars"))
00462 return -1;
00463 for (int i = 0; i < self->n_output_vars; ++i)
00464 {
00465 assert(self->output_vars[i] < self->n_vars);
00466 }
00467 if (unpack_list_of_ssize_t(update_storage, &self->update_storage, &self->n_updates,
00468 "updates_storage"))
00469 return -1;
00470 return 0;
00471 }
00472 static void set_position_of_error(CLazyLinker * self, int owner_idx)
00473 {
00474 if (self->position_of_error == -1)
00475 {
00476 self->position_of_error = owner_idx;
00477 }
00478 }
00479 static PyObject * pycall(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
00480 {
00481 // call thunk to see which inputs it wants
00482 PyObject * thunk = PyList_GetItem(self->thunks, node_idx);
00483 // refcounting - thunk is borrowed
00484 PyObject * rval = NULL;
00485 if (self->do_timing)
00486 {
00487 double t0 = pytime(NULL);
00488 if (verbose) fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
00489 rval = PyObject_CallObject(thunk, NULL);
00490 if (rval)
00491 {
00492 double t1 = pytime(NULL);
00493 double ti = PyFloat_AsDouble(
00494 PyList_GetItem(self->call_times, node_idx));
00495 PyList_SetItem(self->call_times, node_idx,
00496 PyFloat_FromDouble(t1 - t0 + ti));
00497 PyObject * count = PyList_GetItem(self->call_counts, node_idx);
00498 long icount = PyInt_AsLong(count);
00499 PyList_SetItem(self->call_counts, node_idx,
00500 PyInt_FromLong(icount + 1));
00501 }
00502 }
00503 else
00504 {
00505 if (verbose)
00506 {
00507 fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
00508 }
00509 rval = PyObject_CallObject(thunk, NULL);
00510 }
00511 return rval;
00512 }
00513 static int c_call(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
00514 {
00515 void * ptr_addr = self->thunk_cptr_fn[node_idx];
00516 int (*fn)(void*) = (int (*)(void*))(ptr_addr);
00517 if (verbose) fprintf(stderr, "calling non-lazy shortcut (node %i)\n", (int)node_idx);
00518 int err = 0;
00519 if (self->do_timing)
00520 {
00521 double t0 = pytime(NULL);
00522 err = fn(self->thunk_cptr_data[node_idx]);
00523 double t1 = pytime(NULL);
00524 double ti = PyFloat_AsDouble(PyList_GetItem(self->call_times, node_idx));
00525 PyList_SetItem(self->call_times, node_idx, PyFloat_FromDouble(t1 - t0 + ti));
00526 PyObject * count = PyList_GetItem(self->call_counts, node_idx);
00527 long icount = PyInt_AsLong(count);
00528 PyList_SetItem(self->call_counts, node_idx, PyInt_FromLong(icount+1));
00529 }
00530 else
00531 {
00532 err = fn(self->thunk_cptr_data[node_idx]);
00533 }
00534
00535 if (err)
00536 {
00537 // cast the argument to a PyList (as described near line 226 of cc.py)
00538 PyObject * __ERROR = ((PyObject**)self->thunk_cptr_data[node_idx])[0];
00539 assert (PyList_Check(__ERROR));
00540 assert (PyList_Size(__ERROR) == 3);
00541 PyObject * err_type = PyList_GetItem(__ERROR, 0); //stolen ref
00542 PyObject * err_msg = PyList_GetItem(__ERROR, 1); //stolen ref
00543 PyObject * err_trace = PyList_GetItem(__ERROR, 2); //stolen ref
00544 PyList_SET_ITEM(__ERROR, 0, Py_None); Py_INCREF(Py_None); //clobbers old ref
00545 PyList_SET_ITEM(__ERROR, 1, Py_None); Py_INCREF(Py_None); //clobbers old ref
00546 PyList_SET_ITEM(__ERROR, 2, Py_None); Py_INCREF(Py_None); //clobbers old ref
00547
00548 assert(!PyErr_Occurred()); // because CLinker hid the exception in __ERROR aka data
00549 PyErr_Restore(err_type, err_msg, err_trace); //steals refs to args
00550 }
00551 if (err) set_position_of_error(self, node_idx);
00552 return err;
00553 }
00554 static
00555 int lazy_rec_eval(CLazyLinker * self, Py_ssize_t var_idx, PyObject*one, PyObject*zero)
00556 {
00557 PyObject *rval = NULL;
00558 int verbose = 0;
00559 int err = 0;
00560
00561 if (verbose) fprintf(stderr, "lazy_rec computing %i\n", (int)var_idx);
00562
00563 if (self->var_computed[var_idx] || !self->var_has_owner[var_idx])
00564 return 0;
00565
00566 Py_ssize_t owner_idx = self->var_owner[var_idx];
00567
00568 // STEP 1: compute the pre-requirements of the node
00569 // Includes input nodes for non-lazy ops.
00570 for (int i = 0; i < self->node_n_prereqs[owner_idx]; ++i)
00571 {
00572 Py_ssize_t prereq_idx = self->node_prereqs[owner_idx][i];
00573 if (!self->var_computed[prereq_idx])
00574 {
00575 err = lazy_rec_eval(self, prereq_idx, one, zero);
00576 if (err) return err;
00577 }
00578 assert (self->var_computed[prereq_idx]);
00579 }
00580
00581 // STEP 2: compute the node itself
00582 if (self->is_lazy[owner_idx])
00583 {
00584 // update the compute_map cells corresponding to the inputs of this thunk
00585 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00586 {
00587 int in_idx = self->node_inputs[owner_idx][i];
00588 if (self->var_computed[in_idx])
00589 {
00590 Py_INCREF(one);
00591 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, one);
00592 }
00593 else
00594 {
00595 Py_INCREF(zero);
00596 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, zero);
00597 }
00598 if (err) goto fail;
00599 }
00600
00601 rval = pycall(self, owner_idx, verbose);
00602 // refcounting - rval is new ref
00603 //TODO: to prevent infinite loops
00604 // - consider check that a thunk does not ask for an input that is already computed
00605 if (rval == NULL)
00606 {
00607 assert (PyErr_Occurred());
00608 err = 1;
00609 goto fail;
00610 }
00611
00612 //update the computed-ness of any output cells
00613 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00614 {
00615 int out_idx = self->node_outputs[owner_idx][i];
00616 PyObject * el_i = PyList_GetItem(self->var_computed_cells[out_idx], 0);
00617 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00618 if (PyErr_Occurred())
00619 {
00620 err = -1;
00621 goto pyfail;
00622 }
00623 assert (N==0 || N==1);
00624 self->var_computed[out_idx] = N;
00625 }
00626 if (!self->var_computed[var_idx])
00627 {
00628 /*
00629 * If self is not computed after the call, this means that some
00630 * inputs are needed. Compute the ones on the returned list
00631 * and try to compute the current node again (with recursive call).
00632 * This allows a node to request more nodes more than once before
00633 * finally yielding a result.
00634 */
00635 if (!PyList_Check(rval))
00636 {
00637 //TODO: More helpful error to help find *which node* made this
00638 // bad thunk
00639 PyErr_SetString(PyExc_TypeError,
00640 "lazy thunk should return a list");
00641 err = 1;
00642 goto pyfail;
00643 }
00644
00645 if (!PyList_Size(rval))
00646 {
00647 PyErr_SetString(PyExc_ValueError,
00648 "lazy thunk returned empty list without computing output");
00649 err = 1;
00650 goto pyfail;
00651 }
00652
00653 for (int i = 0; i < PyList_Size(rval); ++i)
00654 {
00655 PyObject * el_i = PyList_GetItem(rval, i);
00656 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00657 if (PyErr_Occurred())
00658 {
00659 err = 1;
00660 goto pyfail;
00661 }
00662 assert (N <= self->node_n_inputs[owner_idx]);
00663 Py_ssize_t input_idx = self->node_inputs[owner_idx][N];
00664 err = lazy_rec_eval(self, input_idx, one, zero);
00665 if (err) goto pyfail;
00666 }
00667
00668 Py_DECREF(rval);
00669 /*
00670 * We intentionally skip all the end-of-function processing
00671 * (mark outputs, GC) as it will be performed by the call
00672 * that actually manages to compute the result.
00673 */
00674 return lazy_rec_eval(self, var_idx, one, zero);
00675 }
00676
00677 Py_DECREF(rval);
00678 }
00679 else //owner is not a lazy op. Ensure all intputs are evaluated.
00680 {
00681 // loop over inputs to owner
00682 // call lazy_rec_eval on each one that is not computed.
00683 // if there's an error, pass it up the stack
00684 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00685 {
00686 Py_ssize_t input_idx = self->node_inputs[owner_idx][i];
00687 if (!self->var_computed[input_idx])
00688 {
00689 err = lazy_rec_eval(self, input_idx, one, zero);
00690 if (err) return err;
00691 }
00692 assert (self->var_computed[input_idx]);
00693 }
00694
00695 // call the thunk for this owner.
00696 if (self->thunk_cptr_fn[owner_idx])
00697 {
00698 err = c_call(self, owner_idx, verbose);
00699 if (err) goto fail;
00700 }
00701 else
00702 {
00703 rval = pycall(self, owner_idx, verbose);
00704 //rval is new ref
00705 if (rval) //pycall returned normally (no exception)
00706 {
00707 if (rval == Py_None)
00708 {
00709 Py_DECREF(rval); //ignore a return of None
00710 }
00711 else if (PyList_Check(rval))
00712 {
00713 PyErr_SetString(PyExc_TypeError,
00714 "non-lazy thunk should return None, not list");
00715 err = 1;
00716 goto pyfail;
00717 }
00718 else // don't know what it returned, but it wasn't right.
00719 {
00720 PyErr_SetObject(PyExc_TypeError, rval);
00721 err = 1;
00722 // We don't release rval since we put it in the error above
00723 goto fail;
00724 }
00725 }
00726 else // pycall returned NULL (internal error)
00727 {
00728 err = 1;
00729 goto fail;
00730 }
00731 }
00732 }
00733
00734 // loop over all outputs and mark them as computed
00735 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00736 {
00737 self->var_computed[self->node_outputs[owner_idx][i]] = 1;
00738 }
00739
00740 // Free vars that are not needed anymore
00741 if (self->allow_gc)
00742 {
00743 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00744 {
00745 int cleanup = 1;
00746 Py_ssize_t i_idx = self->node_inputs[owner_idx][i];
00747 if (!self->var_has_owner[i_idx])
00748 continue;
00749
00750 for (int j = 0; j < self->n_output_vars; ++j)
00751 {
00752 if (i_idx == self->output_vars[j])
00753 {
00754 cleanup = 0;
00755 break;
00756 }
00757 }
00758 if (!cleanup) continue;
00759
00760 for (int j = 0; j < self->n_dependencies[i_idx]; ++j)
00761 {
00762 if (!self->var_computed[self->dependencies[i_idx][j]])
00763 {
00764 cleanup = 0;
00765 break;
00766 }
00767 }
00768 if (!cleanup) continue;
00769
00770 Py_INCREF(Py_None);
00771 err = PyList_SetItem(self->var_value_cells[i_idx], 0, Py_None);
00772 //See the Stack gc implementation for why we change it to 2 and not 0.
00773 self->var_computed[i_idx] = 2;
00774 if (err) goto fail;
00775 }
00776 }
00777
00778 return 0;
00779 pyfail:
00780 Py_DECREF(rval);
00781 fail:
00782 set_position_of_error(self, owner_idx);
00783 return err;
00784 }
00785 PyObject *
00786 CLazyLinker_call(PyObject *_self, PyObject *args, PyObject *kwds)
00787 {
00788 CLazyLinker * self = (CLazyLinker*)_self;
00789 static char *kwlist[] = {
00790 (char*)"time_thunks",
00791 (char *)"n_calls",
00792 NULL};
00793 int n_calls=1;
00794 if (! PyArg_ParseTupleAndKeywords(args, kwds, "|ii", kwlist,
00795 &self->do_timing,
00796 &n_calls))
00797 return NULL;
00798 int err = 0;
00799 self->position_of_error = -1;
00800 // create constants used to fill the var_compute_cells
00801 PyObject * one = PyInt_FromLong(1);
00802 PyObject * zero = PyInt_FromLong(0);
00803
00804 // pre-allocate our return value
00805 Py_INCREF(Py_None);
00806 PyObject * rval = Py_None;
00807 //clear storage of pre_call_clear elements
00808 for (int call_i = 0; call_i < n_calls && (!err); ++call_i)
00809 {
00810 Py_ssize_t n_pre_call_clear = PyList_Size(self->pre_call_clear);
00811 assert(PyList_Check(self->pre_call_clear));
00812 for (int i = 0; i < n_pre_call_clear; ++i)
00813 {
00814 PyObject * el_i = PyList_GetItem(self->pre_call_clear, i);
00815 Py_INCREF(Py_None);
00816 PyList_SetItem(el_i, 0, Py_None);
00817 }
00818 //clear the computed flag out of all non-input vars
00819 for (int i = 0; i < self->n_vars; ++i)
00820 {
00821 self->var_computed[i] = !self->var_has_owner[i];
00822 if (self->var_computed[i])
00823 {
00824 Py_INCREF(one);
00825 PyList_SetItem(self->var_computed_cells[i], 0, one);
00826 }
00827 else
00828 {
00829 Py_INCREF(zero);
00830 PyList_SetItem(self->var_computed_cells[i], 0, zero);
00831 }
00832 }
00833
00834 for (int i = 0; i < self->n_output_vars && (!err); ++i)
00835 {
00836 err = lazy_rec_eval(self, self->output_vars[i], one, zero);
00837 }
00838
00839 if (!err)
00840 {
00841 // save references to outputs prior to updating storage containers
00842 assert (self->n_output_vars >= self->n_updates);
00843 Py_DECREF(rval);
00844 rval = PyList_New(self->n_output_vars);
00845 for (int i = 0; i < (self->n_output_vars); ++i)
00846 {
00847 Py_ssize_t src = self->output_vars[i];
00848 PyObject * item = PyList_GetItem(self->var_value_cells[src], 0);
00849 if (self->var_computed[src] != 1)
00850 {
00851 err = 1;
00852 PyErr_Format(PyExc_AssertionError,
00853 "The compute map of output %d should contain "
00854 "1 at the end of execution, not %d.",
00855 i, self->var_computed[src]);
00856 break;
00857 }
00858 Py_INCREF(item);
00859 PyList_SetItem(rval, i, item);
00860 }
00861 }
00862
00863 if (!err)
00864 {
00865 // Update the inputs that have an update rule
00866 for (int i = 0; i < self->n_updates; ++i)
00867 {
00868 PyObject* tmp = PyList_GetItem(rval, self->n_output_vars - self->n_updates + i);
00869 Py_INCREF(tmp);
00870 Py_ssize_t dst = self->update_storage[i];
00871 PyList_SetItem(self->var_value_cells[dst], 0, tmp);
00872 }
00873 }
00874 }
00875
00876 /*
00877 Clear everything that is left and not an output. This is needed
00878 for lazy evaluation since the current GC algo is too conservative
00879 with lazy graphs.
00880 */
00881 if (self->allow_gc && !err)
00882 {
00883 for (Py_ssize_t i = 0; i < self->n_vars; ++i)
00884 {
00885 int do_cleanup = 1;
00886 if (!self->var_has_owner[i] || !self->var_computed[i])
00887 continue;
00888 for (int j = 0; j < self->n_output_vars; ++j)
00889 {
00890 if (i == self->output_vars[j])
00891 {
00892 do_cleanup = 0;
00893 break;
00894 }
00895 }
00896 if (!do_cleanup)
00897 continue;
00898 Py_INCREF(Py_None);
00899 PyList_SetItem(self->var_value_cells[i], 0, Py_None);
00900 }
00901 }
00902 Py_DECREF(one);
00903 Py_DECREF(zero);
00904 if (err)
00905 {
00906 Py_DECREF(rval);
00907 return NULL;
00908 }
00909 return rval;
00910 }
00911
00912 #if 0
00913 static PyMethodDef CLazyLinker_methods[] = {
00914 {
00915 //"name", (PyCFunction)CLazyLinker_accept, METH_VARARGS, "Return the name, combining the first and last name"
00916 },
00917 {NULL} /* Sentinel */
00918 };
00919 #endif
00920
00921 static PyMemberDef CLazyLinker_members[] = {
00922 {(char*)"nodes", T_OBJECT_EX, offsetof(CLazyLinker, nodes), 0,
00923 (char*)"list of nodes"},
00924 {(char*)"thunks", T_OBJECT_EX, offsetof(CLazyLinker, thunks), 0,
00925 (char*)"list of thunks in program"},
00926 {(char*)"call_counts", T_OBJECT_EX, offsetof(CLazyLinker, call_counts), 0,
00927 (char*)"number of calls of each thunk"},
00928 {(char*)"call_times", T_OBJECT_EX, offsetof(CLazyLinker, call_times), 0,
00929 (char*)"total runtime in each thunk"},
00930 {(char*)"position_of_error", T_INT, offsetof(CLazyLinker, position_of_error), 0,
00931 (char*)"position of failed thunk"},
00932 {(char*)"time_thunks", T_INT, offsetof(CLazyLinker, do_timing), 0,
00933 (char*)"bool: nonzero means call will time thunks"},
00934 {(char*)"need_update_inputs", T_INT, offsetof(CLazyLinker, need_update_inputs), 0,
00935 (char*)"bool: nonzero means Function.__call__ must implement update mechanism"},
00936 {NULL} /* Sentinel */
00937 };
00938
00939 static PyTypeObject lazylinker_ext_CLazyLinkerType = {
00940 PyObject_HEAD_INIT(NULL)
00941 0, /*ob_size*/
00942 "lazylinker_ext.CLazyLinker", /*tp_name*/
00943 sizeof(CLazyLinker), /*tp_basicsize*/
00944 0, /*tp_itemsize*/
00945 CLazyLinker_dealloc, /*tp_dealloc*/
00946 0, /*tp_print*/
00947 0, /*tp_getattr*/
00948 0, /*tp_setattr*/
00949 0, /*tp_compare*/
00950 0, /*tp_repr*/
00951 0, /*tp_as_number*/
00952 0, /*tp_as_sequence*/
00953 0, /*tp_as_mapping*/
00954 0, /*tp_hash */
00955 CLazyLinker_call, /*tp_call*/
00956 0, /*tp_str*/
00957 0, /*tp_getattro*/
00958 0, /*tp_setattro*/
00959 0, /*tp_as_buffer*/
00960 Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
00961 "CLazyLinker object", /* tp_doc */
00962 0, /* tp_traverse */
00963 0, /* tp_clear */
00964 0, /* tp_richcompare */
00965 0, /* tp_weaklistoffset */
00966 0, /* tp_iter */
00967 0, /* tp_iternext */
00968 0,//CLazyLinker_methods, /* tp_methods */
00969 CLazyLinker_members, /* tp_members */
00970 0, /* tp_getset */
00971 0, /* tp_base */
00972 0, /* tp_dict */
00973 0, /* tp_descr_get */
00974 0, /* tp_descr_set */
00975 0, /* tp_dictoffset */
00976 (initproc)CLazyLinker_init,/* tp_init */
00977 0, /* tp_alloc */
00978 CLazyLinker_new, /* tp_new */
00979 };
00980
00981 static PyObject * get_version(PyObject *dummy, PyObject *args)
00982 {
00983 PyObject *result = PyFloat_FromDouble(0.20);
00984 return result;
00985 }
00986
00987 static PyMethodDef lazylinker_ext_methods[] = {
00988 {"get_version", get_version, METH_VARARGS, "Get extension version."},
00989 {NULL, NULL, 0, NULL} /* Sentinel */
00990 };
00991
00992 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
00993 #define PyMODINIT_FUNC void
00994 #endif
00995 PyMODINIT_FUNC
00996 initlazylinker_ext(void)
00997 {
00998 PyObject* m;
00999
01000 lazylinker_ext_CLazyLinkerType.tp_new = PyType_GenericNew;
01001 if (PyType_Ready(&lazylinker_ext_CLazyLinkerType) < 0)
01002 return;
01003
01004 m = Py_InitModule3("lazylinker_ext", lazylinker_ext_methods,
01005 "Example module that creates an extension type.");
01006
01007 Py_INCREF(&lazylinker_ext_CLazyLinkerType);
01008 PyModule_AddObject(m, "CLazyLinker", (PyObject *)&lazylinker_ext_CLazyLinkerType);
01009 }
01010
01011
Problem occurred during compilation with the command line below:
g++ -dynamiclib -g -D NPY_ARRAY_ENSURECOPY=NPY_ENSURECOPY -D NPY_ARRAY_ALIGNED=NPY_ALIGNED -D NPY_ARRAY_WRITEABLE=NPY_WRITEABLE -D NPY_ARRAY_UPDATE_ALL=NPY_UPDATE_ALL -D NPY_ARRAY_C_CONTIGUOUS=NPY_C_CONTIGUOUS -D NPY_ARRAY_F_CONTIGUOUS=NPY_F_CONTIGUOUS -fPIC -undefined dynamic_lookup -m64 -I/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/core/include -I/Applications/Canopy.app/appdata/canopy-1.0.0.1160.macosx-x86_64/Canopy.app/Contents/include/python2.7 -o /Users/rkeisler/.theano/compiledir_Darwin-10.8.0-x86_64-i386-64bit-i386-2.7.3/lazylinker_ext/lazylinker_ext.so /Users/rkeisler/.theano/compiledir_Darwin-10.8.0-x86_64-i386-64bit-i386-2.7.3/lazylinker_ext/mod.cpp -L/Applications/Canopy.app/appdata/canopy-1.0.0.1160.macosx-x86_64/Canopy.app/Contents/lib -lpython2.7
===============================
===============================
ld: library not found for -lpython2.7
collect2: ld returned 1 exit status
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/__init__.py", line 61, in <module>
import compile
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/__init__.py", line 6, in <module>
import function_module
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/function_module.py", line 18, in <module>
import mode as mode_module
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/mode.py", line 11, in <module>
import theano.gof.vm
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/vm.py", line 487, in <module>
import lazylinker_c
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/lazylinker_c.py", line 88, in <module>
preargs=args)
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/cmodule.py", line 1633, in compile_str
(status, compile_stderr.replace('\n', '. ')))
Exception: Compilation failed (return status=1): ld: library not found for -lpython2.7. collect2: ld returned 1 exit status.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment