Skip to content

Instantly share code, notes, and snippets.

@jberkus
Created July 27, 2015 22:39
Show Gist options
  • Save jberkus/132a2be7096b1953d72b to your computer and use it in GitHub Desktop.
Save jberkus/132a2be7096b1953d72b to your computer and use it in GitHub Desktop.
Sample script to pull a stream of San Francisco tweets and push them into PipelineDB
from TwitterAPI import TwitterAPI
import psycopg2
from psycopg2.extras import Json
CONSUMER_KEY = 'Get'
CONSUMER_SECRET = 'your'
ACCESS_TOKEN_KEY = 'own'
ACCESS_TOKEN_SECRET = 'API key'
api = TwitterAPI(CONSUMER_KEY,
CONSUMER_SECRET,
ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
# stream all tweets identified as being in San Francisco
r = api.request('statuses/filter', {'locations': '-122.75,36.8,-121.75,37.8'} )
conn = psycopg2.connect('dbname=twitter user=pipeline
password=pipeline host=pipeline port=6543')
conn.autocommit = True
cur = conn.cursor()
for item in r:
# check if this is a tweet by looking for message text
if 'text' in item:
cur.execute("""INSERT INTO tweets ( content ) VALUES ( %s )""",(Json(item),))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment