Initial YQL pipe loader mechanism

pipeloadYQL.py

"""Pipe finders and loaders

   Use these to import Yahoo! pipes as Python modules with support for nested imports

   (C) Greg Gaughan, 2012
"""

"""Example usage (see __main__ below for a dynamic version):

    import sys
    import pipeloadYQL

    #Hooks
    sys.path_hooks.append(pipeloadYQL.PipeYQLFinder)
    sys.path.append('http://query.yahooapis.com/v1/public/yql')
    
    #Import
    import pipe_e6b016246bed2b99958867e85bf1a390  #will load from disk if available; if not, then via YQL

    #Run
    context = Context()
    p = pipe_e6b016246bed2b99958867e85bf1a390.pipe_e6b016246bed2b99958867e85bf1a390(context, None)
    for i in p:
        print i
"""

"""
   This is an initial proof of concept
   
   If this works we can create a base class and then easily add similar loaders for:
       pipeGAE - load json from GAE table/col
       pipeSQL - load json from DBAPI table/col
       etc.?
   And update compile.py to share the base loading logic
"""

import sys
import os
import imp

from pipe2py import Context
import pipe2py.compile

try:
    import json
    json.loads # test access to the attributes of the right json module
except (ImportError, AttributeError):
    import simplejson as json

import urllib


class PipeYQLFinder(object):
    """Find modules from Yahoo via YQL."""
    
    def __init__(self, path_entry):
        if path_entry != "http://query.yahooapis.com/v1/public/yql":
            raise ImportError()
        self.path_entry = path_entry
        return
        
    def __unicode__(self):
        return '<%s for "%s">' % (self.__class__.__name__, self.path_entry)
        
    def find_module(self, fullname, path=None):
        path = path or self.path_entry
        #todo lightweight check if it exists, for now assume pipe_* is valid
        if fullname.startswith("pipe_"):
            return PipeYQLLoader(path)
        return None

class PipeYQLLoader(object):
    """Load source for modules from YQL json."""
    
    def __init__(self, path_entry):
        self.path_entry = path_entry
        self.context = Context()
        return
        
    def _get_filename(self, fullname):
        #Fake filename, so get_data() works correctly
        return os.path.join(self.path_entry, fullname)
        
    def get_source(self, fullname):
        """Load json definition and compile to source code"""
        #Load json definition from Yahoo!
        try:
            url = (self.path_entry +
                   """?q=select%20PIPE.working%20from%20json%20"""
                   """where%20url%3D%22http%3A%2F%2Fpipes.yahoo.com%2Fpipes%2Fpipe.info%3F_out%3Djson%26_id%3D"""
                   + fullname[5:] + 
                   """%22&format=json""")
            pjson = urllib.urlopen(url).readlines()
        except Exception, e: #todo catch not found specifically
            raise ImportError("Could not find source for %s (%s)" % (fullname, unicode(e)))
        pjson = "".join(pjson)
        pipe_def = json.loads(pjson)
        if not pipe_def['query']['results']:
            raise ImportError("Could not find source for %s" % fullname)
        pjson = pipe_def['query']['results']['json']['PIPE']['working']
        pipe_def = pjson  #was json.loads(pjson) until April 2011 - changes at Yahoo! Pipes/YQL?
        #Compile
        p = pipe2py.compile.parse_and_write_pipe(self.context, pipe_def, pipe_name=fullname)
        return p

    def get_code(self, fullname):
        source = self.get_source(fullname)
        #todo exec/build?
        return source
    
    # def get_data(self, path):
        
    def is_package(self, fullname):
        return False

    def load_module(self, fullname):
        source = self.get_source(fullname)

        creating = False
        if fullname in sys.modules:
            #reuse existing module from previous import
            mod = sys.modules[fullname]
        else:
            #create a new module object
            creating = True
            mod = sys.modules.setdefault(fullname, imp.new_module(fullname))

        #Set properties required by PEP 302
        mod.__file__ = self._get_filename(fullname)
        mod.__name__ = fullname
        mod.__loader__ = self
        mod.__package__ = '.'.join(fullname.split('.')[:-1])
        
        if self.is_package(fullname):
            mod.__path__ = [ self.path_entry ]
        
        try:
            exec(source, mod.__dict__)
        except:
            if creating:
                del sys.modules[fullname]
        
        return mod
    
if __name__ == '__main__':
    #Hooks
    sys.path_hooks.append(PipeYQLFinder)
    sys.path.append('http://query.yahooapis.com/v1/public/yql')

    #Dynamic import
    pmod = __import__("pipe_e6b016246bed2b99958867e85bf1a390")

    #Find method
    pm = getattr(pmod, "pipe_e6b016246bed2b99958867e85bf1a390")
    
    #Call & print
    context = Context()
    p = pm(context, None)
    for i in p:
        print i