Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
twitter-analytics-data-scraper
// 1. Go to https://analytics.twitter.com/
// 2. Keep scrolling till the end until all the stats data is loaded
// 3. Right click on the page click on last option "Inspect" a window should open select console from that
// 4. copy this entire function
function getVal (val) {
val=val.replace(/\,/g,'');
multiplier = val.substr(-1).toLowerCase();
if (multiplier == "k")
return parseFloat(val) * 1000;
else if (multiplier == "m")
return parseFloat(val) * 1000000;
else return (val !== '-' ? parseFloat(val) : 0);
}
function dataMiner(){
let masterArray = [];
const totalSummary = document.getElementsByClassName("home-columns").length;
for(let j=0; j< totalSummary; j++){
let tweets = 0;
let tweetviews = 0;
let profileViews = 0;
let mentions = 0;
let followers = 0;
let title = "";
let followPercentage = "0%";
let avgTweetPerDay = 0;
let topFollower = "-";
let item = document.getElementsByClassName("home-columns")[j];
if(!!item.querySelector(".metric-tweets")){
tweets = !!item.querySelector(".metric-tweets").textContent ? item.querySelector(".metric-tweets").textContent : null;
tweets = getVal(tweets);
}
if(!!item.querySelector(".metric-tweetviews")){
tweetviews = !!item.querySelector(".metric-tweetviews").textContent ? item.querySelector(".metric-tweetviews").textContent : null;
tweetviews = getVal(tweetviews);
}
if(!!item.querySelector(".metric-profile-views")){
profileViews = !!item.querySelector(".metric-profile-views").textContent ? item.querySelector(".metric-profile-views").textContent : null;
profileViews = getVal(profileViews);
}
if(!!item.querySelector(".metric-mentions")){
mentions = !!item.querySelector(".metric-mentions").textContent ? item.querySelector(".metric-mentions").textContent: null;
mentions = getVal(mentions);
}
if(!!item.querySelector(".metric-followers")){
followers = !!item.querySelector(".metric-followers").textContent ? item.querySelector(".metric-followers").textContent : null;
followers = getVal(followers);
}
followPercentage = (followers / profileViews * 100).toFixed(2) + "%";
if(followPercentage === "Infinity%" || followPercentage === "NaN%" || followPercentage === "-Infinity%"){
followPercentage = "0%"
}
// avgTweetPerDay
avgTweetPerDay = Math.floor(tweets/30);
// top follower for month
if(!!item.querySelector(".profile-card-name-link")){
topFollower = item.querySelector(".profile-card-name-link").innerText;
}
if(!!item.querySelectorAll('.home-group-header')){
const totalLength = item.querySelectorAll('.home-group-header').length;
title = item.querySelectorAll('.home-group-header')[totalLength - 1].innerHTML;
}
masterArray.push({tweets, tweetviews, profileViews, mentions, followers, followPercentage, topFollower, avgTweetPerDay, title })
}
// to view data in table format
console.log(console.table(masterArray))
// to view data as a JSON String
console.log(JSON.stringify(masterArray))
};
// 5. Hit Enter
// 6. type in console => dataMiner()
// 7. Hit Enter
// 8. You will see output in console as a JSON object, copy that entire object and paste it into https://json-csv.com/
// and convert it csv if you ever want it
// 9. Import this CSV to airtable if you want to save it further
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment