-
-
Save amitdeshmukh/fa1c0fa4774e96dde62fd2fef5b499d0 to your computer and use it in GitHub Desktop.
Download a JSON List of twitter bookmarks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
the twitter api is stupid. it is stupid and bad and expensive. hence, this. | |
Literally just paste this in the JS console on the bookmarks tab and the script will automatically scroll to the bottom of your bookmarks and keep a track of them as it goes. | |
When finished, it downloads a JSON file containing the raw text content of every bookmark. | |
for now it stores just the text inside the tweet itself, but if you're reading this why don't you go ahead and try to also store other information (author, tweetLink, pictures, everything). come on. do it. please? | |
*/ | |
let tweets = []; // Initialize an empty array to hold all tweet elements | |
let domChangeCount = 0; // Initialize a counter for the number of DOM changes | |
const maxDomChanges = 50; | |
const scrollInterval = 1000; | |
const scrollStep = 5000; // Pixels to scroll on each step | |
let previousTweetCount = 0; | |
let unchangedCount = 0; | |
const scrollToEndIntervalID = setInterval(() => { | |
window.scrollBy(0, scrollStep); | |
const currentTweetCount = tweets.length; | |
if (currentTweetCount === previousTweetCount) { | |
unchangedCount++; | |
if (unchangedCount >= 2) { // Stop if the count has not changed 5 times | |
console.log('Scraping complete'); | |
console.log('Total tweets scraped: ', tweets.length); | |
console.log('Downloading tweets as JSON...'); | |
clearInterval(scrollToEndIntervalID); // Stop scrolling | |
observer.disconnect(); // Stop observing DOM changes | |
downloadTweetsAsJson(tweets); // Download the tweets list as a JSON file | |
} | |
} else { | |
unchangedCount = 0; // Reset counter if new tweets were added | |
} | |
previousTweetCount = currentTweetCount; // Update previous count for the next check | |
}, scrollInterval); | |
function updateTweets() { | |
document.querySelectorAll('article[data-testid="tweet"]').forEach(tweetElement => { | |
let authorName = tweetElement.querySelector('[data-testid="User-Name"]')?.innerText.split('\n')[0]; | |
const handle = tweetElement.querySelector('[role="link"]').href.split('/').pop(); | |
const tweetText = tweetElement.querySelector('[data-testid="tweetText"]')?.innerText; | |
const time = tweetElement.querySelector('time').getAttribute('datetime'); | |
const tweetLink = tweetElement.querySelector('a[href*="/status/"]')?.href; | |
const replies = tweetElement.querySelector('[data-testid="reply"]')?.innerText.split(' ')[0]; | |
const retweets = tweetElement.querySelector('[data-testid="retweet"]')?.innerText.split(' ')[0]; | |
const likes = tweetElement.querySelector('[data-testid="like"]')?.innerText.split(' ')[0]; | |
// Extract images | |
const images = Array.from(tweetElement.querySelectorAll('div[data-testid="tweetPhoto"] img')).map(img => img.src); | |
// Extract videos | |
const videos = Array.from(tweetElement.querySelectorAll('video')).map(video => video.src); | |
// Extract GIFs | |
const gifs = Array.from(tweetElement.querySelectorAll('[data-testid="tweetPhoto"] img')).filter(img => img.src.includes('tweet_video_thumb')).map(img => img.src); | |
// Extract external links | |
const externalLinks = Array.from(tweetElement.querySelectorAll('[data-testid="tweetText"] a')).map(a => a.href); | |
const isTweetNew = !tweets.some(tweet => tweet.tweetText === tweetText); | |
if (isTweetNew) { | |
tweets.push({ | |
authorName, | |
handle, | |
tweetText, | |
time, | |
tweetLink, | |
replies, | |
retweets, | |
likes, | |
media: { | |
images, | |
videos, | |
gifs, | |
externalLinks | |
} | |
}); | |
console.log("tweets scraped: ", tweets.length); | |
} | |
}); | |
} | |
// Initially populate the tweets array | |
updateTweets(); | |
// Create a MutationObserver to observe changes in the DOM | |
const observer = new MutationObserver(mutations => { | |
mutations.forEach(mutation => { | |
if (mutation.addedNodes.length) { | |
updateTweets(); // Call updateTweets whenever new nodes are added to the DOM | |
domChangeCount++; // Increment the counter for DOM changes | |
if (domChangeCount >= maxDomChanges) { | |
observer.disconnect(); // Stop observing DOM changes | |
} | |
} | |
}); | |
}); | |
// Start observing the document body for child list changes | |
observer.observe(document.body, { childList: true, subtree: true }); | |
function downloadTweetsAsJson(tweetsArray) { | |
const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON | |
const blob = new Blob([jsonData], { type: 'application/json' }); | |
const url = URL.createObjectURL(blob); | |
const link = document.createElement('a'); | |
link.href = url; | |
link.download = 'tweets.json'; // Specify the file name | |
document.body.appendChild(link); // Append the link to the document | |
link.click(); // Programmatically click the link to trigger the download | |
document.body.removeChild(link); // Clean up and remove the link | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment