process.py

process.py

from twisted.internet import protocol
from twisted.internet import reactor
import re

class MyPP(protocol.ProcessProtocol):
    def __init__(self, verses):
        self.verses = verses
        self.data = ""
    def connectionMade(self):
        print "connectionMade!"
        
    def outReceived(self, data):
        print "outReceived! with %d bytes!" % len(data)
        self.data = self.data + data
    def errReceived(self, data):
        print "errReceived! with %d bytes!" % len(data)
    def inConnectionLost(self):
        print "inConnectionLost! stdin is closed! (we probably did it)"
    def outConnectionLost(self):
        print "outConnectionLost! The child closed their stdout!"
        # now is the time to examine what they wrote
        #print "I saw them write:", self.data
        (dummy, lines, words, chars, file) = re.split(r'\s+', self.data)
        print "I saw %s lines" % lines
    def errConnectionLost(self):
        print "errConnectionLost! The child closed their stderr."
    def processExited(self, reason):
        print "processExited, status %d" % (reason.value.exitCode,)
    def processEnded(self, reason):
        print "processEnded, status %d" % (reason.value.exitCode,)
        print "quitting"
        reactor.stop()

class StreamingSpider(Spider, protocol.ProcessProtocol):
    name = "streaming"
    allowed_domains = []
    start_urls = ('www.google.com',
        )
    def __init__(self, Input=None, *args, **kwargs):
        super(StreamingSpider, self).__init__(*args, **kwargs)
        pp = MyPP(10)
        reactor.spawnProcess(pp, "python", ["python", str(Input)], {})
        reactor.run()
        pass


    def parse(self, response):
        self.transport.write(str(response.url))
        self.transport.closeStdin() # tell them we're done
        pass