This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Tweet(Base): | |
__tablename__ = "tweets" | |
id = Column('id', Integer, primary_key=True) | |
user = Column('username', String) | |
date = Column('date', String) | |
likes = Column('likes', Integer) | |
retweets = Column('retweets', Integer) | |
comments = Column('comments', Integer) | |
text = Column('text', String) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Base = declarative_base() | |
engine = create_engine("YOUR URI", echo=True) | |
Session = sessionmaker(bind=engine) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sqlalchemy import create_engine, Column, Integer, String | |
from sqlalchemy.ext.declarative import declarative_base | |
from sqlalchemy.orm import sessionmaker |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __scrap_tweets(self): | |
sleep(2) | |
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]") | |
for tweet in tweets: | |
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text | |
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 " | |
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text | |
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]") | |
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __format_data(self, data): | |
if data.__contains__("K"): | |
return int(float(data.replace("K", "")) * 1000) | |
if data.__contains__("M"): | |
return int(float(data.replace("M", "")) * 10000000) | |
return int(data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class TBot: | |
def __init__(self): | |
self.driver = webdriver.Chrome() | |
self.driver.get("https://twitter.com/login") | |
self.__login("pablotcampos25@gmail.com", "AreYouRellyExpectingMyPswd?") | |
while True: | |
self.__scrap_tweets() | |
self.__scroll() | |
def __login(self, username, pswd): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __scrap_tweets(self): | |
try: | |
sleep(2) | |
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]") | |
for tweet in tweets: | |
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text | |
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 " | |
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text | |
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]") | |
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __scroll(self): | |
self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight)") | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class TBot: | |
def __init__(self): | |
self.driver = webdriver.Chrome() | |
self.driver.get("https://twitter.com/login") | |
self.__login("pablotcampos25@gmail.com", "ThisIsStillNotMyPassword") | |
self.__scrap_tweets() | |
while True: | |
pass | |
def __login(self, username, pswd): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __scrap_tweets(self): | |
sleep(2) | |
tweets = self.driver.find_elements_by_xpath("//article[@role=\"article\"]") | |
for tweet in tweets: | |
author = tweet.find_element_by_xpath(".//div[@dir=\"ltr\"]").text | |
text = tweet.find_element_by_xpath(".//div[@class=\"css-901oao r-hkyrab r-1qd0xha r-a023e6 r-16dba41 " | |
"r-ad9z0x r-bcqeeo r-bnwqim r-qvutc0\"]").text | |
data = tweet.find_elements_by_xpath(".//div[@class=\"css-1dbjc4n r-xoduu5 r-1udh08x\"]") | |
time = tweet.find_element_by_xpath(".//time").get_attribute("datetime") |