Skip to content

Instantly share code, notes, and snippets.

@martingaido
Last active November 30, 2022 21:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save martingaido/b555189c1ce354359837c79ab25c0c1a to your computer and use it in GitHub Desktop.
Save martingaido/b555189c1ce354359837c79ab25c0c1a to your computer and use it in GitHub Desktop.
AWS Lambda Function to Scrape Tweets
import json
import snscrape
import snscrape.modules.twitter as sntwitter
import snscrape.modules.facebook as snfacebbok
import snscrape.modules.instagram as sninstagram
import snscrape.modules.mastodon as snmastodon
import snscrape.modules.reddit as smreddit
def lambda_handler(event, context):
# Available Methods for Twitter
#
# TwitterSearchScraper
# TwitterUserScraper
# TwitterProfileScraper
# TwitterHashtagScraper
# TwitterTweetScraperMode
# TwitterTweetScraper
# TwitterListPostsScraper
# TwitterTrendsScraper
if event['Mode'] == 'user':
scraper = sntwitter.TwitterUserScraper(event['Handler'])
elif event['Mode'] == 'hashtag':
scraper = sntwitter.TwitterHashtagScraper(event['Hashtag'])
elif event['Mode'] == 'trend':
scraper = sntwitter.TwitterTrendsScraper(event['Trend'])
else:
scraper = sntwitter.TwitterSearchScraper(event['Hashtag'])
tweets = []
for i, tweet in enumerate(scraper.get_items()):
data = [
tweet.date,
tweet.id,
tweet.lang,
tweet.source,
tweet.content,
tweet.user.username,
tweet.likeCount,
tweet.retweetCount,
tweet.replyCount,
]
tweets.append(data)
if i >= event['MaxResults'] - 1:
break
return {
'statusCode': 200,
'body': json.dumps(tweets, default=str),
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment