Skip to content

Instantly share code, notes, and snippets.

@joerussbowman
Created August 27, 2011 23:29
Show Gist options
  • Save joerussbowman/1176002 to your computer and use it in GitHub Desktop.
Save joerussbowman/1176002 to your computer and use it in GitHub Desktop.
Tornado Twitter Stream
# not pretty, I quickly moved to another idea
# not involving the Twitter stream. Good starting
# point.
import re
import base64
import socket
import asyncmongo
from tornado import ioloop
from tornado import iostream
from tornado import escape
#substitue your Twitter username/password below
auth = base64.b64encode("USERNAME:PASSWORD")
db = asyncmongo.Client(pool_id='mydb', host='127.0.0.1', port=27017, maxcached=10, maxconnections=50, dbname='trends')
def send_request():
stream.write("GET /1/statuses/sample.json HTTP/1.0\r\nHost: stream.twitter.com\r\nAuthorization: Basic %s\r\n\r\n" % auth)
stream.read_until("\r\n\r\n", on_headers)
def on_headers(data):
headers = {}
for line in data.split("\r\n"):
parts = line.split(":")
if len(parts) == 2:
headers[parts[0].strip()] = parts[1].strip()
stream.read_until("\n", on_body)
def quit(*args, **kwargs):
stream.close()
ioloop.IOLoop.instance().stop()
def cb_pass(*args, **kwargs):
pass
def on_body(data):
tweet = escape.json_decode(data)
text = re.sub("@\w+", "", tweet.get("text", ""))
text = re.sub("RT", "", text)
text = re.sub("http\w+", "", text)
if text != "":
#print text
db.tweets.insert({"text": text}, callback=cb_pass)
stream.read_until("\n", on_body)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(("stream.twitter.com", 80), send_request)
ioloop.IOLoop.instance().start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment