Last active
November 30, 2022 21:33
-
-
Save martingaido/b555189c1ce354359837c79ab25c0c1a to your computer and use it in GitHub Desktop.
AWS Lambda Function to Scrape Tweets
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import snscrape | |
import snscrape.modules.twitter as sntwitter | |
import snscrape.modules.facebook as snfacebbok | |
import snscrape.modules.instagram as sninstagram | |
import snscrape.modules.mastodon as snmastodon | |
import snscrape.modules.reddit as smreddit | |
def lambda_handler(event, context): | |
# Available Methods for Twitter | |
# | |
# TwitterSearchScraper | |
# TwitterUserScraper | |
# TwitterProfileScraper | |
# TwitterHashtagScraper | |
# TwitterTweetScraperMode | |
# TwitterTweetScraper | |
# TwitterListPostsScraper | |
# TwitterTrendsScraper | |
if event['Mode'] == 'user': | |
scraper = sntwitter.TwitterUserScraper(event['Handler']) | |
elif event['Mode'] == 'hashtag': | |
scraper = sntwitter.TwitterHashtagScraper(event['Hashtag']) | |
elif event['Mode'] == 'trend': | |
scraper = sntwitter.TwitterTrendsScraper(event['Trend']) | |
else: | |
scraper = sntwitter.TwitterSearchScraper(event['Hashtag']) | |
tweets = [] | |
for i, tweet in enumerate(scraper.get_items()): | |
data = [ | |
tweet.date, | |
tweet.id, | |
tweet.lang, | |
tweet.source, | |
tweet.content, | |
tweet.user.username, | |
tweet.likeCount, | |
tweet.retweetCount, | |
tweet.replyCount, | |
] | |
tweets.append(data) | |
if i >= event['MaxResults'] - 1: | |
break | |
return { | |
'statusCode': 200, | |
'body': json.dumps(tweets, default=str), | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment