public
anonymous / gist:5548936
Created

output of python -c "import theano".

  • Download Gist
gistfile1.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
00001 #include <Python.h>
00002 #include "structmember.h"
00003 #include <sys/time.h>
00004
00005 // Old Python compatibility from here:
00006 // http://www.python.org/dev/peps/pep-0353/
00007 #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
00008 typedef int Py_ssize_t;
00009 #define PY_SSIZE_T_MAX INT_MAX
00010 #define PY_SSIZE_T_MIN INT_MIN
00011 // This one was taken from:
00012 // http://svn.python.org/projects/python/trunk/Modules/_ctypes/ctypes.h
00013 #define PyNumber_AsSsize_t(ob, exc) PyInt_AsLong(ob)
00014 #endif
00015
00016 /**
00017
00018 TODO:
00019 - Check max supported depth of recursion
00020 - CLazyLinker should add context information to errors caught during evaluation. Say what node we were on, add the traceback attached to the node.
00021 - Clear containers of fully-useed intermediate results if allow_gc is 1
00022 - Add timers for profiling
00023 - Add support for profiling space used.
00024
00025
00026 */
00027 static double pytime(const struct timeval * tv)
00028 {
00029 struct timeval t;
00030 if (!tv)
00031 {
00032 tv = &t;
00033 gettimeofday(&t, NULL);
00034 }
00035 return (double) tv->tv_sec + (double) tv->tv_usec / 1000000.0;
00036 }
00037
00038 /**
00039 Helper routine to convert a PyList of integers to a c array of integers.
00040 */
00041 static int unpack_list_of_ssize_t(PyObject * pylist, Py_ssize_t **dst, Py_ssize_t *len,
00042 const char* kwname)
00043 {
00044 Py_ssize_t buflen, *buf;
00045 if (!PyList_Check(pylist))
00046 {
00047 PyErr_Format(PyExc_TypeError, "%s must be list", kwname);
00048 return -1;
00049 }
00050 assert (NULL == *dst);
00051 *len = buflen = PyList_Size(pylist);
00052 *dst = buf = (Py_ssize_t*)calloc(buflen, sizeof(Py_ssize_t));
00053 assert(buf);
00054 for (int ii = 0; ii < buflen; ++ii)
00055 {
00056 PyObject * el_i = PyList_GetItem(pylist, ii);
00057 Py_ssize_t n_i = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00058 if (PyErr_Occurred())
00059 {
00060 free(buf);
00061 *dst = NULL;
00062 return -1;
00063 }
00064 buf[ii] = n_i;
00065 }
00066 return 0;
00067 }
00068
00069 /**
00070
00071 CLazyLinker
00072
00073
00074 */
00075 typedef struct {
00076 PyObject_HEAD
00077 /* Type-specific fields go here. */
00078 PyObject * nodes; // the python list of nodes
00079 PyObject * thunks; // python list of thunks
00080 PyObject * pre_call_clear; //list of cells to clear on call.
00081 int allow_gc;
00082 Py_ssize_t n_applies;
00083 int n_vars; // number of variables in the graph
00084 int * var_computed; // 1 or 0 for every variable
00085 PyObject ** var_computed_cells;
00086 PyObject ** var_value_cells;
00087 Py_ssize_t **dependencies; // list of vars dependencies for GC
00088 Py_ssize_t *n_dependencies;
00089
00090 Py_ssize_t n_output_vars;
00091 Py_ssize_t * output_vars; // variables that *must* be evaluated by call
00092
00093 int * is_lazy; // 1 or 0 for every thunk
00094
00095 Py_ssize_t * var_owner; // nodes[[var_owner[var_idx]]] is var[var_idx]->owner
00096 int * var_has_owner; // 1 or 0
00097
00098 Py_ssize_t * node_n_inputs;
00099 Py_ssize_t * node_n_outputs;
00100 Py_ssize_t ** node_inputs;
00101 Py_ssize_t ** node_outputs;
00102 Py_ssize_t * node_inputs_outputs_base; // node_inputs and node_outputs point into this
00103 Py_ssize_t * node_n_prereqs;
00104 Py_ssize_t ** node_prereqs;
00105
00106 Py_ssize_t * update_storage; // input cells to update with the last outputs in output_vars
00107 Py_ssize_t n_updates;
00108
00109 void ** thunk_cptr_fn;
00110 void ** thunk_cptr_data;
00111 PyObject * call_times;
00112 PyObject * call_counts;
00113 int do_timing;
00114 int need_update_inputs;
00115 int position_of_error; // -1 for no error, otw the index into `thunks` that failed.
00116 } CLazyLinker;
00117
00118
00119 static void
00120 CLazyLinker_dealloc(PyObject* _self)
00121 {
00122 CLazyLinker* self = (CLazyLinker *) _self;
00123 free(self->thunk_cptr_fn);
00124 free(self->thunk_cptr_data);
00125
00126 free(self->is_lazy);
00127
00128 free(self->update_storage);
00129
00130 if (self->node_n_prereqs)
00131 {
00132 for (int i = 0; i < self->n_applies; ++i)
00133 {
00134 free(self->node_prereqs[i]);
00135 }
00136 }
00137 free(self->node_n_prereqs);
00138 free(self->node_prereqs);
00139 free(self->node_inputs_outputs_base);
00140 free(self->node_n_inputs);
00141 free(self->node_n_outputs);
00142 free(self->node_inputs);
00143 free(self->node_outputs);
00144
00145 if (self->dependencies)
00146 {
00147 for (int i = 0; i < self->n_vars; ++i)
00148 {
00149 free(self->dependencies[i]);
00150 }
00151 free(self->dependencies);
00152 free(self->n_dependencies);
00153 }
00154
00155 free(self->var_owner);
00156 free(self->var_has_owner);
00157 free(self->var_computed);
00158 if (self->var_computed_cells)
00159 {
00160 for (int i = 0; i < self->n_vars; ++i)
00161 {
00162 Py_DECREF(self->var_computed_cells[i]);
00163 Py_DECREF(self->var_value_cells[i]);
00164 }
00165 }
00166 free(self->var_computed_cells);
00167 free(self->var_value_cells);
00168 free(self->output_vars);
00169
00170 Py_XDECREF(self->nodes);
00171 Py_XDECREF(self->thunks);
00172 Py_XDECREF(self->call_times);
00173 Py_XDECREF(self->call_counts);
00174 Py_XDECREF(self->pre_call_clear);
00175 self->ob_type->tp_free((PyObject*)self);
00176 }
00177 static PyObject *
00178 CLazyLinker_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
00179 {
00180 CLazyLinker *self;
00181
00182 self = (CLazyLinker *)type->tp_alloc(type, 0);
00183 if (self != NULL) {
00184 self->nodes = NULL;
00185 self->thunks = NULL;
00186 self->pre_call_clear = NULL;
00187
00188 self->allow_gc = 1;
00189 self->n_applies = 0;
00190 self->n_vars = 0;
00191 self->var_computed = NULL;
00192 self->var_computed_cells = NULL;
00193 self->var_value_cells = NULL;
00194 self->dependencies = NULL;
00195 self->n_dependencies = NULL;
00196
00197 self->n_output_vars = 0;
00198 self->output_vars = NULL;
00199
00200 self->is_lazy = NULL;
00201
00202 self->var_owner = NULL;
00203 self->var_has_owner = NULL;
00204
00205 self->node_n_inputs = NULL;
00206 self->node_n_outputs = NULL;
00207 self->node_inputs = NULL;
00208 self->node_outputs = NULL;
00209 self->node_inputs_outputs_base = NULL;
00210 self->node_prereqs = NULL;
00211 self->node_n_prereqs = NULL;
00212
00213 self->update_storage = NULL;
00214 self->n_updates = 0;
00215
00216 self->thunk_cptr_data = NULL;
00217 self->thunk_cptr_fn = NULL;
00218 self->call_times = NULL;
00219 self->call_counts = NULL;
00220 self->do_timing = 0;
00221
00222 self->need_update_inputs = 0;
00223 self->position_of_error = -1;
00224 }
00225 return (PyObject *)self;
00226 }
00227
00228 static int
00229 CLazyLinker_init(CLazyLinker *self, PyObject *args, PyObject *kwds)
00230 {
00231 static char *kwlist[] = {
00232 (char*)"nodes",
00233 (char*)"thunks",
00234 (char*)"pre_call_clear",
00235 (char*)"allow_gc",
00236 (char*)"call_counts",
00237 (char*)"call_times",
00238 (char*)"compute_map_list",
00239 (char*)"storage_map_list",
00240 (char*)"base_input_output_list",
00241 (char*)"node_n_inputs",
00242 (char*)"node_n_outputs",
00243 (char*)"node_input_offset",
00244 (char*)"node_output_offset",
00245 (char*)"var_owner",
00246 (char*)"is_lazy_list",
00247 (char*)"output_vars",
00248 (char*)"node_prereqs",
00249 (char*)"node_output_size",
00250 (char*)"update_storage",
00251 (char*)"dependencies",
00252 NULL};
00253
00254 PyObject *compute_map_list=NULL,
00255 *storage_map_list=NULL,
00256 *base_input_output_list=NULL,
00257 *node_n_inputs=NULL,
00258 *node_n_outputs=NULL,
00259 *node_input_offset=NULL,
00260 *node_output_offset=NULL,
00261 *var_owner=NULL,
00262 *is_lazy=NULL,
00263 *output_vars=NULL,
00264 *node_prereqs=NULL,
00265 *node_output_size=NULL,
00266 *update_storage=NULL,
00267 *dependencies=NULL;
00268
00269 assert(!self->nodes);
00270 if (! PyArg_ParseTupleAndKeywords(args, kwds, "OOOiOOOOOOOOOOOOOOOO", kwlist,
00271 &self->nodes,
00272 &self->thunks,
00273 &self->pre_call_clear,
00274 &self->allow_gc,
00275 &self->call_counts,
00276 &self->call_times,
00277 &compute_map_list,
00278 &storage_map_list,
00279 &base_input_output_list,
00280 &node_n_inputs,
00281 &node_n_outputs,
00282 &node_input_offset,
00283 &node_output_offset,
00284 &var_owner,
00285 &is_lazy,
00286 &output_vars,
00287 &node_prereqs,
00288 &node_output_size,
00289 &update_storage,
00290 &dependencies
00291 ))
00292 return -1;
00293 Py_INCREF(self->nodes);
00294 Py_INCREF(self->thunks);
00295 Py_INCREF(self->pre_call_clear);
00296 Py_INCREF(self->call_counts);
00297 Py_INCREF(self->call_times);
00298
00299 Py_ssize_t n_applies = PyList_Size(self->nodes);
00300
00301 self->n_applies = n_applies;
00302 self->n_vars = PyList_Size(var_owner);
00303
00304 if (PyList_Size(self->thunks) != n_applies) return -1;
00305 if (PyList_Size(self->call_counts) != n_applies) return -1;
00306 if (PyList_Size(self->call_times) != n_applies) return -1;
00307
00308 // allocated and initialize thunk_cptr_data and thunk_cptr_fn
00309 if (n_applies)
00310 {
00311 self->thunk_cptr_data = (void**)calloc(n_applies, sizeof(void*));
00312 self->thunk_cptr_fn = (void**)calloc(n_applies, sizeof(void*));
00313 self->is_lazy = (int*)calloc(n_applies, sizeof(int));
00314 self->node_prereqs = (Py_ssize_t**)calloc(n_applies, sizeof(Py_ssize_t*));
00315 self->node_n_prereqs = (Py_ssize_t*)calloc(n_applies, sizeof(Py_ssize_t));
00316 assert(self->node_prereqs);
00317 assert(self->node_n_prereqs);
00318 assert(self->is_lazy);
00319 assert(self->thunk_cptr_fn);
00320 assert(self->thunk_cptr_data);
00321
00322 for (int i = 0; i < n_applies; ++i)
00323 {
00324 PyObject * thunk = PyList_GetItem(self->thunks, i);
00325 //thunk is borrowed
00326 if (PyObject_HasAttrString(thunk, "cthunk"))
00327 {
00328 PyObject * cthunk = PyObject_GetAttrString(thunk, "cthunk");
00329 //new reference
00330 assert (cthunk && PyCObject_Check(cthunk));
00331 self->thunk_cptr_fn[i] = PyCObject_AsVoidPtr(cthunk);
00332 self->thunk_cptr_data[i] = PyCObject_GetDesc(cthunk);
00333 Py_DECREF(cthunk);
00334 // cthunk is kept alive by membership in self->thunks
00335 }
00336
00337 PyObject * el_i = PyList_GetItem(is_lazy, i);
00338 self->is_lazy[i] = PyNumber_AsSsize_t(el_i, NULL);
00339
00340 /* now get the prereqs */
00341 el_i = PyList_GetItem(node_prereqs, i);
00342 assert (PyList_Check(el_i));
00343 self->node_n_prereqs[i] = PyList_Size(el_i);
00344 if (self->node_n_prereqs[i])
00345 {
00346 self->node_prereqs[i] = (Py_ssize_t*)malloc(
00347 PyList_Size(el_i)*sizeof(Py_ssize_t));
00348 for (int j = 0; j < PyList_Size(el_i); ++j)
00349 {
00350 PyObject * el_ij = PyList_GetItem(el_i, j);
00351 Py_ssize_t N = PyNumber_AsSsize_t(el_ij, PyExc_IndexError);
00352 if (PyErr_Occurred())
00353 return -1;
00354 // N < n. variables
00355 assert(N < PyList_Size(var_owner));
00356 self->node_prereqs[i][j] = N;
00357 }
00358 }
00359 }
00360 }
00361 if (PyList_Check(base_input_output_list))
00362 {
00363 Py_ssize_t n_inputs_outputs_base = PyList_Size(base_input_output_list);
00364 self->node_inputs_outputs_base = (Py_ssize_t*)calloc(n_inputs_outputs_base,sizeof(Py_ssize_t));
00365 assert(self->node_inputs_outputs_base);
00366 for (int i = 0; i < n_inputs_outputs_base; ++i)
00367 {
00368 PyObject *el_i = PyList_GetItem(base_input_output_list, i);
00369 Py_ssize_t idx = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00370 if (PyErr_Occurred()) return -1;
00371 self->node_inputs_outputs_base[i] = idx;
00372 }
00373 self->node_n_inputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
00374 assert(self->node_n_inputs);
00375 self->node_n_outputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
00376 assert(self->node_n_outputs);
00377 self->node_inputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
00378 assert(self->node_inputs);
00379 self->node_outputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
00380 assert(self->node_outputs);
00381 for (int i = 0; i < n_applies; ++i)
00382 {
00383 Py_ssize_t N;
00384 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_inputs, i),PyExc_IndexError);
00385 if (PyErr_Occurred()) return -1;
00386 assert (N <= n_inputs_outputs_base);
00387 self->node_n_inputs[i] = N;
00388 N = PyNumber_AsSsize_t(PyList_GetItem(node_n_outputs, i),PyExc_IndexError);
00389 if (PyErr_Occurred()) return -1;
00390 assert (N <= n_inputs_outputs_base);
00391 self->node_n_outputs[i] = N;
00392 N = PyNumber_AsSsize_t(PyList_GetItem(node_input_offset, i),PyExc_IndexError);
00393 if (PyErr_Occurred()) return -1;
00394 assert (N <= n_inputs_outputs_base);
00395 self->node_inputs[i] = &self->node_inputs_outputs_base[N];
00396 N = PyNumber_AsSsize_t(PyList_GetItem(node_output_offset, i),PyExc_IndexError);
00397 if (PyErr_Occurred()) return -1;
00398 assert (N <= n_inputs_outputs_base);
00399 self->node_outputs[i] = &self->node_inputs_outputs_base[N];
00400 }
00401 }
00402 else
00403 {
00404 PyErr_SetString(PyExc_TypeError, "base_input_output_list must be list");
00405 return -1;
00406 }
00407
00408 // allocation for var_owner
00409 if (PyList_Check(var_owner))
00410 {
00411 self->var_owner = (Py_ssize_t*)calloc(self->n_vars,sizeof(Py_ssize_t));
00412 self->var_has_owner = (int*)calloc(self->n_vars,sizeof(int));
00413 self->var_computed = (int*)calloc(self->n_vars,sizeof(int));
00414 self->var_computed_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
00415 self->var_value_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
00416 for (int i = 0; i < self->n_vars; ++i)
00417 {
00418 PyObject * el_i = PyList_GetItem(var_owner, i);
00419 if (el_i == Py_None)
00420 {
00421 self->var_has_owner[i] = 0;
00422 }
00423 else
00424 {
00425 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00426 if (PyErr_Occurred()) return -1;
00427 assert (N <= n_applies);
00428 self->var_owner[i] = N;
00429 self->var_has_owner[i] = 1;
00430 }
00431 self->var_computed_cells[i] = PyList_GetItem(compute_map_list, i);
00432 Py_INCREF(self->var_computed_cells[i]);
00433 self->var_value_cells[i] = PyList_GetItem(storage_map_list, i);
00434 Py_INCREF(self->var_value_cells[i]);
00435 }
00436 }
00437 else
00438 {
00439 PyErr_SetString(PyExc_TypeError, "var_owner must be list");
00440 return -1;
00441 }
00442
00443 if (dependencies != Py_None)
00444 {
00445 self->dependencies = (Py_ssize_t**)calloc(self->n_vars, sizeof(Py_ssize_t *));
00446 self->n_dependencies = (Py_ssize_t*)calloc(self->n_vars, sizeof(Py_ssize_t));
00447 assert(self->dependencies);
00448 assert(self->n_dependencies);
00449
00450 for (int i = 0; i < self->n_vars; ++i)
00451 {
00452 PyObject *tmp = PyList_GetItem(dependencies, i);
00453 // refcounting - tmp is borrowed
00454 if (unpack_list_of_ssize_t(tmp, &self->dependencies[i], &self->n_dependencies[i],
00455 "dependencies"))
00456 return -1;
00457 }
00458 }
00459
00460 if (unpack_list_of_ssize_t(output_vars, &self->output_vars, &self->n_output_vars,
00461 "output_vars"))
00462 return -1;
00463 for (int i = 0; i < self->n_output_vars; ++i)
00464 {
00465 assert(self->output_vars[i] < self->n_vars);
00466 }
00467 if (unpack_list_of_ssize_t(update_storage, &self->update_storage, &self->n_updates,
00468 "updates_storage"))
00469 return -1;
00470 return 0;
00471 }
00472 static void set_position_of_error(CLazyLinker * self, int owner_idx)
00473 {
00474 if (self->position_of_error == -1)
00475 {
00476 self->position_of_error = owner_idx;
00477 }
00478 }
00479 static PyObject * pycall(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
00480 {
00481 // call thunk to see which inputs it wants
00482 PyObject * thunk = PyList_GetItem(self->thunks, node_idx);
00483 // refcounting - thunk is borrowed
00484 PyObject * rval = NULL;
00485 if (self->do_timing)
00486 {
00487 double t0 = pytime(NULL);
00488 if (verbose) fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
00489 rval = PyObject_CallObject(thunk, NULL);
00490 if (rval)
00491 {
00492 double t1 = pytime(NULL);
00493 double ti = PyFloat_AsDouble(
00494 PyList_GetItem(self->call_times, node_idx));
00495 PyList_SetItem(self->call_times, node_idx,
00496 PyFloat_FromDouble(t1 - t0 + ti));
00497 PyObject * count = PyList_GetItem(self->call_counts, node_idx);
00498 long icount = PyInt_AsLong(count);
00499 PyList_SetItem(self->call_counts, node_idx,
00500 PyInt_FromLong(icount + 1));
00501 }
00502 }
00503 else
00504 {
00505 if (verbose)
00506 {
00507 fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
00508 }
00509 rval = PyObject_CallObject(thunk, NULL);
00510 }
00511 return rval;
00512 }
00513 static int c_call(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
00514 {
00515 void * ptr_addr = self->thunk_cptr_fn[node_idx];
00516 int (*fn)(void*) = (int (*)(void*))(ptr_addr);
00517 if (verbose) fprintf(stderr, "calling non-lazy shortcut (node %i)\n", (int)node_idx);
00518 int err = 0;
00519 if (self->do_timing)
00520 {
00521 double t0 = pytime(NULL);
00522 err = fn(self->thunk_cptr_data[node_idx]);
00523 double t1 = pytime(NULL);
00524 double ti = PyFloat_AsDouble(PyList_GetItem(self->call_times, node_idx));
00525 PyList_SetItem(self->call_times, node_idx, PyFloat_FromDouble(t1 - t0 + ti));
00526 PyObject * count = PyList_GetItem(self->call_counts, node_idx);
00527 long icount = PyInt_AsLong(count);
00528 PyList_SetItem(self->call_counts, node_idx, PyInt_FromLong(icount+1));
00529 }
00530 else
00531 {
00532 err = fn(self->thunk_cptr_data[node_idx]);
00533 }
00534
00535 if (err)
00536 {
00537 // cast the argument to a PyList (as described near line 226 of cc.py)
00538 PyObject * __ERROR = ((PyObject**)self->thunk_cptr_data[node_idx])[0];
00539 assert (PyList_Check(__ERROR));
00540 assert (PyList_Size(__ERROR) == 3);
00541 PyObject * err_type = PyList_GetItem(__ERROR, 0); //stolen ref
00542 PyObject * err_msg = PyList_GetItem(__ERROR, 1); //stolen ref
00543 PyObject * err_trace = PyList_GetItem(__ERROR, 2); //stolen ref
00544 PyList_SET_ITEM(__ERROR, 0, Py_None); Py_INCREF(Py_None); //clobbers old ref
00545 PyList_SET_ITEM(__ERROR, 1, Py_None); Py_INCREF(Py_None); //clobbers old ref
00546 PyList_SET_ITEM(__ERROR, 2, Py_None); Py_INCREF(Py_None); //clobbers old ref
00547
00548 assert(!PyErr_Occurred()); // because CLinker hid the exception in __ERROR aka data
00549 PyErr_Restore(err_type, err_msg, err_trace); //steals refs to args
00550 }
00551 if (err) set_position_of_error(self, node_idx);
00552 return err;
00553 }
00554 static
00555 int lazy_rec_eval(CLazyLinker * self, Py_ssize_t var_idx, PyObject*one, PyObject*zero)
00556 {
00557 PyObject *rval = NULL;
00558 int verbose = 0;
00559 int err = 0;
00560
00561 if (verbose) fprintf(stderr, "lazy_rec computing %i\n", (int)var_idx);
00562
00563 if (self->var_computed[var_idx] || !self->var_has_owner[var_idx])
00564 return 0;
00565
00566 Py_ssize_t owner_idx = self->var_owner[var_idx];
00567
00568 // STEP 1: compute the pre-requirements of the node
00569 // Includes input nodes for non-lazy ops.
00570 for (int i = 0; i < self->node_n_prereqs[owner_idx]; ++i)
00571 {
00572 Py_ssize_t prereq_idx = self->node_prereqs[owner_idx][i];
00573 if (!self->var_computed[prereq_idx])
00574 {
00575 err = lazy_rec_eval(self, prereq_idx, one, zero);
00576 if (err) return err;
00577 }
00578 assert (self->var_computed[prereq_idx]);
00579 }
00580
00581 // STEP 2: compute the node itself
00582 if (self->is_lazy[owner_idx])
00583 {
00584 // update the compute_map cells corresponding to the inputs of this thunk
00585 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00586 {
00587 int in_idx = self->node_inputs[owner_idx][i];
00588 if (self->var_computed[in_idx])
00589 {
00590 Py_INCREF(one);
00591 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, one);
00592 }
00593 else
00594 {
00595 Py_INCREF(zero);
00596 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, zero);
00597 }
00598 if (err) goto fail;
00599 }
00600
00601 rval = pycall(self, owner_idx, verbose);
00602 // refcounting - rval is new ref
00603 //TODO: to prevent infinite loops
00604 // - consider check that a thunk does not ask for an input that is already computed
00605 if (rval == NULL)
00606 {
00607 assert (PyErr_Occurred());
00608 err = 1;
00609 goto fail;
00610 }
00611
00612 //update the computed-ness of any output cells
00613 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00614 {
00615 int out_idx = self->node_outputs[owner_idx][i];
00616 PyObject * el_i = PyList_GetItem(self->var_computed_cells[out_idx], 0);
00617 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00618 if (PyErr_Occurred())
00619 {
00620 err = -1;
00621 goto pyfail;
00622 }
00623 assert (N==0 || N==1);
00624 self->var_computed[out_idx] = N;
00625 }
00626 if (!self->var_computed[var_idx])
00627 {
00628 /*
00629 * If self is not computed after the call, this means that some
00630 * inputs are needed. Compute the ones on the returned list
00631 * and try to compute the current node again (with recursive call).
00632 * This allows a node to request more nodes more than once before
00633 * finally yielding a result.
00634 */
00635 if (!PyList_Check(rval))
00636 {
00637 //TODO: More helpful error to help find *which node* made this
00638 // bad thunk
00639 PyErr_SetString(PyExc_TypeError,
00640 "lazy thunk should return a list");
00641 err = 1;
00642 goto pyfail;
00643 }
00644
00645 if (!PyList_Size(rval))
00646 {
00647 PyErr_SetString(PyExc_ValueError,
00648 "lazy thunk returned empty list without computing output");
00649 err = 1;
00650 goto pyfail;
00651 }
00652
00653 for (int i = 0; i < PyList_Size(rval); ++i)
00654 {
00655 PyObject * el_i = PyList_GetItem(rval, i);
00656 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
00657 if (PyErr_Occurred())
00658 {
00659 err = 1;
00660 goto pyfail;
00661 }
00662 assert (N <= self->node_n_inputs[owner_idx]);
00663 Py_ssize_t input_idx = self->node_inputs[owner_idx][N];
00664 err = lazy_rec_eval(self, input_idx, one, zero);
00665 if (err) goto pyfail;
00666 }
00667
00668 Py_DECREF(rval);
00669 /*
00670 * We intentionally skip all the end-of-function processing
00671 * (mark outputs, GC) as it will be performed by the call
00672 * that actually manages to compute the result.
00673 */
00674 return lazy_rec_eval(self, var_idx, one, zero);
00675 }
00676
00677 Py_DECREF(rval);
00678 }
00679 else //owner is not a lazy op. Ensure all intputs are evaluated.
00680 {
00681 // loop over inputs to owner
00682 // call lazy_rec_eval on each one that is not computed.
00683 // if there's an error, pass it up the stack
00684 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00685 {
00686 Py_ssize_t input_idx = self->node_inputs[owner_idx][i];
00687 if (!self->var_computed[input_idx])
00688 {
00689 err = lazy_rec_eval(self, input_idx, one, zero);
00690 if (err) return err;
00691 }
00692 assert (self->var_computed[input_idx]);
00693 }
00694
00695 // call the thunk for this owner.
00696 if (self->thunk_cptr_fn[owner_idx])
00697 {
00698 err = c_call(self, owner_idx, verbose);
00699 if (err) goto fail;
00700 }
00701 else
00702 {
00703 rval = pycall(self, owner_idx, verbose);
00704 //rval is new ref
00705 if (rval) //pycall returned normally (no exception)
00706 {
00707 if (rval == Py_None)
00708 {
00709 Py_DECREF(rval); //ignore a return of None
00710 }
00711 else if (PyList_Check(rval))
00712 {
00713 PyErr_SetString(PyExc_TypeError,
00714 "non-lazy thunk should return None, not list");
00715 err = 1;
00716 goto pyfail;
00717 }
00718 else // don't know what it returned, but it wasn't right.
00719 {
00720 PyErr_SetObject(PyExc_TypeError, rval);
00721 err = 1;
00722 // We don't release rval since we put it in the error above
00723 goto fail;
00724 }
00725 }
00726 else // pycall returned NULL (internal error)
00727 {
00728 err = 1;
00729 goto fail;
00730 }
00731 }
00732 }
00733
00734 // loop over all outputs and mark them as computed
00735 for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
00736 {
00737 self->var_computed[self->node_outputs[owner_idx][i]] = 1;
00738 }
00739
00740 // Free vars that are not needed anymore
00741 if (self->allow_gc)
00742 {
00743 for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
00744 {
00745 int cleanup = 1;
00746 Py_ssize_t i_idx = self->node_inputs[owner_idx][i];
00747 if (!self->var_has_owner[i_idx])
00748 continue;
00749
00750 for (int j = 0; j < self->n_output_vars; ++j)
00751 {
00752 if (i_idx == self->output_vars[j])
00753 {
00754 cleanup = 0;
00755 break;
00756 }
00757 }
00758 if (!cleanup) continue;
00759
00760 for (int j = 0; j < self->n_dependencies[i_idx]; ++j)
00761 {
00762 if (!self->var_computed[self->dependencies[i_idx][j]])
00763 {
00764 cleanup = 0;
00765 break;
00766 }
00767 }
00768 if (!cleanup) continue;
00769
00770 Py_INCREF(Py_None);
00771 err = PyList_SetItem(self->var_value_cells[i_idx], 0, Py_None);
00772 //See the Stack gc implementation for why we change it to 2 and not 0.
00773 self->var_computed[i_idx] = 2;
00774 if (err) goto fail;
00775 }
00776 }
00777
00778 return 0;
00779 pyfail:
00780 Py_DECREF(rval);
00781 fail:
00782 set_position_of_error(self, owner_idx);
00783 return err;
00784 }
00785 PyObject *
00786 CLazyLinker_call(PyObject *_self, PyObject *args, PyObject *kwds)
00787 {
00788 CLazyLinker * self = (CLazyLinker*)_self;
00789 static char *kwlist[] = {
00790 (char*)"time_thunks",
00791 (char *)"n_calls",
00792 NULL};
00793 int n_calls=1;
00794 if (! PyArg_ParseTupleAndKeywords(args, kwds, "|ii", kwlist,
00795 &self->do_timing,
00796 &n_calls))
00797 return NULL;
00798 int err = 0;
00799 self->position_of_error = -1;
00800 // create constants used to fill the var_compute_cells
00801 PyObject * one = PyInt_FromLong(1);
00802 PyObject * zero = PyInt_FromLong(0);
00803
00804 // pre-allocate our return value
00805 Py_INCREF(Py_None);
00806 PyObject * rval = Py_None;
00807 //clear storage of pre_call_clear elements
00808 for (int call_i = 0; call_i < n_calls && (!err); ++call_i)
00809 {
00810 Py_ssize_t n_pre_call_clear = PyList_Size(self->pre_call_clear);
00811 assert(PyList_Check(self->pre_call_clear));
00812 for (int i = 0; i < n_pre_call_clear; ++i)
00813 {
00814 PyObject * el_i = PyList_GetItem(self->pre_call_clear, i);
00815 Py_INCREF(Py_None);
00816 PyList_SetItem(el_i, 0, Py_None);
00817 }
00818 //clear the computed flag out of all non-input vars
00819 for (int i = 0; i < self->n_vars; ++i)
00820 {
00821 self->var_computed[i] = !self->var_has_owner[i];
00822 if (self->var_computed[i])
00823 {
00824 Py_INCREF(one);
00825 PyList_SetItem(self->var_computed_cells[i], 0, one);
00826 }
00827 else
00828 {
00829 Py_INCREF(zero);
00830 PyList_SetItem(self->var_computed_cells[i], 0, zero);
00831 }
00832 }
00833
00834 for (int i = 0; i < self->n_output_vars && (!err); ++i)
00835 {
00836 err = lazy_rec_eval(self, self->output_vars[i], one, zero);
00837 }
00838
00839 if (!err)
00840 {
00841 // save references to outputs prior to updating storage containers
00842 assert (self->n_output_vars >= self->n_updates);
00843 Py_DECREF(rval);
00844 rval = PyList_New(self->n_output_vars);
00845 for (int i = 0; i < (self->n_output_vars); ++i)
00846 {
00847 Py_ssize_t src = self->output_vars[i];
00848 PyObject * item = PyList_GetItem(self->var_value_cells[src], 0);
00849 if (self->var_computed[src] != 1)
00850 {
00851 err = 1;
00852 PyErr_Format(PyExc_AssertionError,
00853 "The compute map of output %d should contain "
00854 "1 at the end of execution, not %d.",
00855 i, self->var_computed[src]);
00856 break;
00857 }
00858 Py_INCREF(item);
00859 PyList_SetItem(rval, i, item);
00860 }
00861 }
00862
00863 if (!err)
00864 {
00865 // Update the inputs that have an update rule
00866 for (int i = 0; i < self->n_updates; ++i)
00867 {
00868 PyObject* tmp = PyList_GetItem(rval, self->n_output_vars - self->n_updates + i);
00869 Py_INCREF(tmp);
00870 Py_ssize_t dst = self->update_storage[i];
00871 PyList_SetItem(self->var_value_cells[dst], 0, tmp);
00872 }
00873 }
00874 }
00875
00876 /*
00877 Clear everything that is left and not an output. This is needed
00878 for lazy evaluation since the current GC algo is too conservative
00879 with lazy graphs.
00880 */
00881 if (self->allow_gc && !err)
00882 {
00883 for (Py_ssize_t i = 0; i < self->n_vars; ++i)
00884 {
00885 int do_cleanup = 1;
00886 if (!self->var_has_owner[i] || !self->var_computed[i])
00887 continue;
00888 for (int j = 0; j < self->n_output_vars; ++j)
00889 {
00890 if (i == self->output_vars[j])
00891 {
00892 do_cleanup = 0;
00893 break;
00894 }
00895 }
00896 if (!do_cleanup)
00897 continue;
00898 Py_INCREF(Py_None);
00899 PyList_SetItem(self->var_value_cells[i], 0, Py_None);
00900 }
00901 }
00902 Py_DECREF(one);
00903 Py_DECREF(zero);
00904 if (err)
00905 {
00906 Py_DECREF(rval);
00907 return NULL;
00908 }
00909 return rval;
00910 }
00911
00912 #if 0
00913 static PyMethodDef CLazyLinker_methods[] = {
00914 {
00915 //"name", (PyCFunction)CLazyLinker_accept, METH_VARARGS, "Return the name, combining the first and last name"
00916 },
00917 {NULL} /* Sentinel */
00918 };
00919 #endif
00920
00921 static PyMemberDef CLazyLinker_members[] = {
00922 {(char*)"nodes", T_OBJECT_EX, offsetof(CLazyLinker, nodes), 0,
00923 (char*)"list of nodes"},
00924 {(char*)"thunks", T_OBJECT_EX, offsetof(CLazyLinker, thunks), 0,
00925 (char*)"list of thunks in program"},
00926 {(char*)"call_counts", T_OBJECT_EX, offsetof(CLazyLinker, call_counts), 0,
00927 (char*)"number of calls of each thunk"},
00928 {(char*)"call_times", T_OBJECT_EX, offsetof(CLazyLinker, call_times), 0,
00929 (char*)"total runtime in each thunk"},
00930 {(char*)"position_of_error", T_INT, offsetof(CLazyLinker, position_of_error), 0,
00931 (char*)"position of failed thunk"},
00932 {(char*)"time_thunks", T_INT, offsetof(CLazyLinker, do_timing), 0,
00933 (char*)"bool: nonzero means call will time thunks"},
00934 {(char*)"need_update_inputs", T_INT, offsetof(CLazyLinker, need_update_inputs), 0,
00935 (char*)"bool: nonzero means Function.__call__ must implement update mechanism"},
00936 {NULL} /* Sentinel */
00937 };
00938
00939 static PyTypeObject lazylinker_ext_CLazyLinkerType = {
00940 PyObject_HEAD_INIT(NULL)
00941 0, /*ob_size*/
00942 "lazylinker_ext.CLazyLinker", /*tp_name*/
00943 sizeof(CLazyLinker), /*tp_basicsize*/
00944 0, /*tp_itemsize*/
00945 CLazyLinker_dealloc, /*tp_dealloc*/
00946 0, /*tp_print*/
00947 0, /*tp_getattr*/
00948 0, /*tp_setattr*/
00949 0, /*tp_compare*/
00950 0, /*tp_repr*/
00951 0, /*tp_as_number*/
00952 0, /*tp_as_sequence*/
00953 0, /*tp_as_mapping*/
00954 0, /*tp_hash */
00955 CLazyLinker_call, /*tp_call*/
00956 0, /*tp_str*/
00957 0, /*tp_getattro*/
00958 0, /*tp_setattro*/
00959 0, /*tp_as_buffer*/
00960 Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/
00961 "CLazyLinker object", /* tp_doc */
00962 0, /* tp_traverse */
00963 0, /* tp_clear */
00964 0, /* tp_richcompare */
00965 0, /* tp_weaklistoffset */
00966 0, /* tp_iter */
00967 0, /* tp_iternext */
00968 0,//CLazyLinker_methods, /* tp_methods */
00969 CLazyLinker_members, /* tp_members */
00970 0, /* tp_getset */
00971 0, /* tp_base */
00972 0, /* tp_dict */
00973 0, /* tp_descr_get */
00974 0, /* tp_descr_set */
00975 0, /* tp_dictoffset */
00976 (initproc)CLazyLinker_init,/* tp_init */
00977 0, /* tp_alloc */
00978 CLazyLinker_new, /* tp_new */
00979 };
00980
00981 static PyObject * get_version(PyObject *dummy, PyObject *args)
00982 {
00983 PyObject *result = PyFloat_FromDouble(0.20);
00984 return result;
00985 }
00986
00987 static PyMethodDef lazylinker_ext_methods[] = {
00988 {"get_version", get_version, METH_VARARGS, "Get extension version."},
00989 {NULL, NULL, 0, NULL} /* Sentinel */
00990 };
00991
00992 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
00993 #define PyMODINIT_FUNC void
00994 #endif
00995 PyMODINIT_FUNC
00996 initlazylinker_ext(void)
00997 {
00998 PyObject* m;
00999
01000 lazylinker_ext_CLazyLinkerType.tp_new = PyType_GenericNew;
01001 if (PyType_Ready(&lazylinker_ext_CLazyLinkerType) < 0)
01002 return;
01003
01004 m = Py_InitModule3("lazylinker_ext", lazylinker_ext_methods,
01005 "Example module that creates an extension type.");
01006
01007 Py_INCREF(&lazylinker_ext_CLazyLinkerType);
01008 PyModule_AddObject(m, "CLazyLinker", (PyObject *)&lazylinker_ext_CLazyLinkerType);
01009 }
01010
01011
Problem occurred during compilation with the command line below:
g++ -dynamiclib -g -D NPY_ARRAY_ENSURECOPY=NPY_ENSURECOPY -D NPY_ARRAY_ALIGNED=NPY_ALIGNED -D NPY_ARRAY_WRITEABLE=NPY_WRITEABLE -D NPY_ARRAY_UPDATE_ALL=NPY_UPDATE_ALL -D NPY_ARRAY_C_CONTIGUOUS=NPY_C_CONTIGUOUS -D NPY_ARRAY_F_CONTIGUOUS=NPY_F_CONTIGUOUS -fPIC -undefined dynamic_lookup -m64 -I/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/core/include -I/Applications/Canopy.app/appdata/canopy-1.0.0.1160.macosx-x86_64/Canopy.app/Contents/include/python2.7 -o /Users/rkeisler/.theano/compiledir_Darwin-10.8.0-x86_64-i386-64bit-i386-2.7.3/lazylinker_ext/lazylinker_ext.so /Users/rkeisler/.theano/compiledir_Darwin-10.8.0-x86_64-i386-64bit-i386-2.7.3/lazylinker_ext/mod.cpp -L/Applications/Canopy.app/appdata/canopy-1.0.0.1160.macosx-x86_64/Canopy.app/Contents/lib -lpython2.7
===============================
===============================
ld: library not found for -lpython2.7
collect2: ld returned 1 exit status
 
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/__init__.py", line 61, in <module>
import compile
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/__init__.py", line 6, in <module>
import function_module
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/function_module.py", line 18, in <module>
import mode as mode_module
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/compile/mode.py", line 11, in <module>
import theano.gof.vm
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/vm.py", line 487, in <module>
import lazylinker_c
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/lazylinker_c.py", line 88, in <module>
preargs=args)
File "/Users/rkeisler/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/theano/gof/cmodule.py", line 1633, in compile_str
(status, compile_stderr.replace('\n', '. ')))
Exception: Compilation failed (return status=1): ld: library not found for -lpython2.7. collect2: ld returned 1 exit status.

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.