gistfile1.txt

gnipclient.py
class PowertrackClient(object):
    """
    Auth attributes common to all gnip clients
    """

    def __init__(self, username, password, account):
        self.username = username
        self.password = password
        self.account = account
        self.auth = (username, password)
class PowertrackStreamingClient(PowertrackClient):
    """
    A single threaded streaming client.
    """

    def __init__(self, *args, **kwargs):
        super(PowertrackStreamingClient, self).__init__(*args, **kwargs)


    def _makeStreamURL(self):
        """
        returns the url to stream from gnip
        """
        return "https://stream.gnip.com:443/accounts/%s/publishers/twitter/streams/track/prod.json" % self.account
        #return "https://stream.gnip.com:443/accounts/%s/publishers/twitter/streams/track/testing.json" % self.account

    def stream(self):
        """
        Establishes a streaming http connection and passes lines to a processor
        """
        resp = requests.get(self._makeStreamURL(), auth=self.auth, stream=True, timeout=120)

        for line in resp.iter_lines():
            self.processData(line)
            
consume.py
from utils import *

class SMTASConsumer(PowertrackStreamingClient):
    def processData(self, data):
        if data.strip() != '':
            process_datapoints.delay(data.strip())

class Command(BaseCommand):
    args = ''
    help = 'Consumes data from gnip'

    def handle(self, *args, **options):
        consumer = SMTASConsumer("username", "pwd", "org")
        consumer.stream()


restart.py

import sys
import os
import time
import subprocess
import psutil


file_path = "/home/administrator/smtas3/manage.py"

class Command(BaseCommand):
    args = ''
    help = 'Consumes data from gnip'

    def handle(self, *args, **options):
        script_name = "consume"
        psutil_check(script_name)

def psutil_check(script_name):
    #has to be changed to the .cmdline instead of .name as multiple python process might exist. done
    state_flag = 0
    for proc in psutil.process_iter():
        #print proc.cmdline
        if len(proc.cmdline) > 3 and proc.cmdline[2].find(script_name) > -1:
            print proc.cmdline, proc.pid, proc.ppid
            state_flag = 1
            break

    if state_flag == 0:
        print "Gnip connection not currently running"
        restart_state = start_gnip()
        if restart_state == 'success':
            state_flag = 1
            print "Gnip connection restarted \n"
        elif restart_state == 'failed':
            print "Gnip connection could not be restarted \n"
        
    elif state_flag == 1:
        print "Gnip connection is currently up\n"


def start_gnip():
    try:
        print "Tring to restart Gnip connection\n"
        cmd = "/home/administrator/.virtualenvs/smtas/bin/python2.7 %s consume --settings='S3.remote' &" % (file_path)
        restart_state = os.system(cmd)
        return "success"
    except Exception, e:
        print >>sys.stderr, "Execution failed:", e
        return "failed"