Skip to content

Instantly share code, notes, and snippets.

class Tweet(Base):
__tablename__ = "tweets"
id = Column('id', Integer, primary_key=True)
user = Column('username', String)
date = Column('date', String)
likes = Column('likes', Integer)
retweets = Column('retweets', Integer)
comments = Column('comments', Integer)
text = Column('text', String)
Base = declarative_base()
engine = create_engine("YOUR URI", echo=True)
Session = sessionmaker(bind=engine)
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
def __scrap_tweets(self):
sleep(2)
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]")
for tweet in tweets:
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 "
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]")
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime")
def __format_data(self, data):
if data.__contains__("K"):
return int(float(data.replace("K", "")) * 1000)
if data.__contains__("M"):
return int(float(data.replace("M", "")) * 10000000)
return int(data)
class TBot:
def __init__(self):
self.driver = webdriver.Chrome()
self.driver.get("https://twitter.com/login")
self.__login("pablotcampos25@gmail.com", "AreYouRellyExpectingMyPswd?")
while True:
self.__scrap_tweets()
self.__scroll()
def __login(self, username, pswd):
def __scrap_tweets(self):
try:
sleep(2)
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]")
for tweet in tweets:
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 "
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]")
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime")
def __scroll(self):
self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
class TBot:
def __init__(self):
self.driver = webdriver.Chrome()
self.driver.get("https://twitter.com/login")
self.__login("pablotcampos25@gmail.com", "ThisIsStillNotMyPassword")
self.__scrap_tweets()
while True:
pass
def __login(self, username, pswd):
@PaburoTC
PaburoTC / TBot.__scrap_tweets.py
Created April 30, 2020 19:46
TBot.__scrap_tweets.py
def __scrap_tweets(self):
sleep(2)
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]")
for tweet in tweets:
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 "
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]")
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime")