Skip to content

Instantly share code, notes, and snippets.

@neelindresh
neelindresh / TwitterStream.py
Last active February 20, 2019 18:52
Getting Twitter Data
import tweepy
from tweepy import Cursor,API
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler,Stream
import Credential
auth=OAuthHandler(Credential.CUSTOMER_KEY,Credential.CUSTOMER_SECRECT_KEY)
auth.set_access_token(Credential.ACCESS_TOKEN,Credential.ACCESS_SECRECT_TOKEN)
api=API(auth,wait_on_rate_limit=True)
import pandas as pd
data=pd.read_csv('location.csv')
print(data.columns)
data=data[['country_name','city_name','country_iso_code']]
print(data)
data.to_csv('city_country_data.csv')
import json
def string_to_json(line):
import ast
jsonval=ast.literal_eval(line)
return jsonval
location=[]
for line in open('tweetCur.json', 'r',encoding="utf-8"):
line=string_to_json(line)
import pandas as pd
import numpy as np
import tqdm
data=pd.read_csv('city_country_data.csv')
df=pd.DataFrame(columns=['Country','code'])
with open('location.txt','r')as r:
location=r.readlines()
for i in tqdm.tqdm(data['country_name'].unique()):
for j in location:
import plotly.offline as py
import pandas as pd
df = pd.read_csv('tweet_location.csv')
count_dict=dict(df['Country'].value_counts())
df['count']=df['Country'].map(count_dict)
df=df.drop(['Unnamed: 0'],axis=1)
df=df.drop_duplicates()
print(df)
CONTRACTION_MAP = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
from bs4 import BeautifulSoup
import urllib3
import json
http=urllib3.PoolManager()
Abbr_dict={}
#Function to get the Slangs from https://www.noslang.com/dictionary/
def getAbbr(alpha):
global Abbr_dict
r=http.request('GET','https://www.noslang.com/dictionary/'+alpha)
soup=BeautifulSoup(r.data,'html.parser')
import json
with open('ShortendText.json','r') as file:
Abbr_dict=json.loads(file.read())
splitLine=line.split()
from textblob import TextBlob
#line == text
blob=TextBlob(line)
blob.correct()
from gingerit.gingerit import GingerIt
parser = GingerIt()
#line==string you wanna correct
tweet=parser.parse(line)