Still trying to get the Rop problem fixed, which might just be masking a linking error. Now I cannot compile the expression graph f = function([W,V,x], JV), I get a cryptic import error (see the tail of this gist).

theano_Rop_exception.py

## my link line: ldflags = -L/scinet/gpc/intel/ics/composer_xe_2011_sp1.9.293/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_scalapack_lp64 -lpthread -lm 

In [1]: import theano.tensor as T
Using gpu device 0: Tesla M2070

In [2]: W = T.dmatrix("W")

In [3]: V = T.dmatrix("V")

In [4]: x = T.dvector('x')

In [5]: y = T.dot(x,W)

In [6]: JV = T.Rop(y,W,V)

In [7]: from theano import function

In [8]: f = function([W,V,x], JV)
---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
/home/z/zhaolei/lzamparo/projects/gpu_tests/<ipython-input-8-c557e1edaeb8> in <module>()
----> 1 f = function([W,V,x], JV)

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/compile/function.pyc in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
    219                 allow_input_downcast=allow_input_downcast,
    220                 on_unused_input=on_unused_input,
--> 221                 profile=profile)
    222     # We need to add the flag check_aliased inputs if we have any mutable or

    223     # borrowed used defined inputs


/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/compile/pfunc.pyc in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
    504     return orig_function(inputs, cloned_outputs, mode,
    505             accept_inplace=accept_inplace, name=name, profile=profile,
--> 506             on_unused_input=on_unused_input)
    507 
    508 

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/compile/function_module.pyc in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input)
   1337                 profile=profile,
   1338                 on_unused_input=on_unused_input).create(
-> 1339                         defaults)
   1340 
   1341     t2 = time.time()

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/compile/function_module.pyc in create(self, input_storage, trustme)
   1165         # Get a function instance

   1166         start_linker = time.time()
-> 1167         _fn, _i, _o = self.linker.make_thunk(input_storage=input_storage_lists)
   1168         end_linker = time.time()
   1169 

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/link.pyc in make_thunk(self, profiler, input_storage, output_storage)
    381         return self.make_all(profiler = profiler,
    382                              input_storage = input_storage,
--> 383                              output_storage = output_storage)[:3]
    384 
    385     def make_all(self, profiler, input_storage, output_storage):

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/vm.pyc in make_all(self, profiler, input_storage, output_storage)
    797                     compute_map,
    798                     no_recycling)
--> 799                         for node in order]
    800 
    801         computed, last_user = link.gc_helper(order)

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/op.pyc in make_thunk(self, node, storage_map, compute_map, no_recycling)
    578                 logger.debug('Trying CLinker.make_thunk')
    579                 outputs = cl.make_thunk(input_storage=node_input_storage,
--> 580                                         output_storage=node_output_storage)
    581                 fill_storage, node_input_filters, node_output_filters = outputs
    582 

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cc.pyc in make_thunk(self, input_storage, output_storage, keep_lock)
    911         cthunk, in_storage, out_storage, error_storage = self.__compile__(
    912             input_storage, output_storage,
--> 913             keep_lock=keep_lock)
    914 
    915         res = _CThunk(cthunk, init_tasks, tasks, error_storage)

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cc.pyc in __compile__(self, input_storage, output_storage, keep_lock)
    854                                     input_storage,
    855                                     output_storage,
--> 856                                     keep_lock=keep_lock)
    857         return (thunk,
    858                 [link.Container(input, storage) for input, storage in

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cc.pyc in cthunk_factory(self, error_storage, in_storage, out_storage, keep_lock)
   1277         else:
   1278             module = get_module_cache().module_from_key(
-> 1279                 key=key, fn=self.compile_cmodule_by_step, keep_lock=keep_lock)
   1280 
   1281         vars = self.inputs + self.outputs + self.orphans

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cmodule.pyc in module_from_key(self, key, fn, keep_lock, key_data)
    973                                 # The module should be returned by the last

    974                                 # step of the compilation.

--> 975                                 module = compile_steps.next()
    976                             except StopIteration:
    977                                 break

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cc.pyc in compile_cmodule_by_step(self, location)
   1199                     lib_dirs=self.lib_dirs(),
   1200                     libs=libs,
-> 1201                     preargs=preargs)
   1202             except Exception, e:
   1203                 e.args += (str(self.fgraph),)

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cmodule.pyc in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, py_module)
   1639             #touch the __init__ file

   1640             file(os.path.join(location, "__init__.py"), 'w').close()
-> 1641             return dlimport(lib_filename)
   1642 
   1643 

/home/z/zhaolei/lzamparo/lib/python2.7/site-packages/theano/gof/cmodule.pyc in dlimport(fullpath, suffix)
    261     sys.path[0:0] = [workdir]  # insert workdir at beginning (temporarily)
    262     try:
--> 263         rval = __import__(module_name, {}, {}, [module_name])
    264         if not rval:
    265             raise Exception('__import__ failed', fullpath)

ImportError: ('/scinet/gpc/intel/ics/composer_xe_2011_sp1.9.293/mkl/lib/intel64/libmkl_scalapack_lp64.so: undefined symbol: Csgsum2d', '[CGemv{inplace}(<TensorType(float64, vector)>, TensorConstant{1.0}, V.T, x, TensorConstant{0.0})]')

Update: test now works fine with the following link line:

print theano.config.blas.ldflags
-L/scinet/gpc/intel/ics/composer_xe_2011_sp1.9.293/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lpthread -lm

Will pass this along to the theano people.