twitter-analytics-data-scraper

dataMiner.js

// 1. Go to https://analytics.twitter.com/ 

// 2. Keep scrolling till the end until all the stats data is loaded

// 3. Right click on the page click on last option "Inspect" a window should open select console from that

// 4. copy this entire function

function getVal (val) {
    val=val.replace(/\,/g,'');
  multiplier = val.substr(-1).toLowerCase();
  if (multiplier == "k")
    return parseFloat(val) * 1000;
  else if (multiplier == "m")
    return parseFloat(val) * 1000000;
   else return (val !== '-' ? parseFloat(val) : 0);
}

function dataMiner(){  
        let masterArray = [];
        const totalSummary = document.getElementsByClassName("home-columns").length;
        for(let j=0; j< totalSummary; j++){
            let tweets = 0;
            let tweetviews = 0;
            let profileViews = 0;
            let mentions = 0;
            let followers = 0;
            let title = "";
            let followPercentage = "0%";
            let avgTweetPerDay = 0;
            let topFollower = "-";
            let item = document.getElementsByClassName("home-columns")[j];
            if(!!item.querySelector(".metric-tweets")){
                tweets = !!item.querySelector(".metric-tweets").textContent ? item.querySelector(".metric-tweets").textContent : null;
                tweets = getVal(tweets);
            }

            if(!!item.querySelector(".metric-tweetviews")){
                tweetviews = !!item.querySelector(".metric-tweetviews").textContent ? item.querySelector(".metric-tweetviews").textContent : null;
                tweetviews = getVal(tweetviews);
            }

            if(!!item.querySelector(".metric-profile-views")){
                profileViews = !!item.querySelector(".metric-profile-views").textContent ? item.querySelector(".metric-profile-views").textContent : null;
                profileViews = getVal(profileViews);
            }

            if(!!item.querySelector(".metric-mentions")){
                mentions = !!item.querySelector(".metric-mentions").textContent ? item.querySelector(".metric-mentions").textContent: null;
                mentions = getVal(mentions);
            }

            if(!!item.querySelector(".metric-followers")){
                followers = !!item.querySelector(".metric-followers").textContent ? item.querySelector(".metric-followers").textContent : null;
                followers = getVal(followers);
            }

            followPercentage = (followers / profileViews * 100).toFixed(2) + "%";
            if(followPercentage === "Infinity%" || followPercentage === "NaN%" || followPercentage === "-Infinity%"){
                followPercentage = "0%"
            }

            // avgTweetPerDay
            avgTweetPerDay = Math.floor(tweets/30);

            // top follower for month
            if(!!item.querySelector(".profile-card-name-link")){
                topFollower = item.querySelector(".profile-card-name-link").innerText;
            }

            if(!!item.querySelectorAll('.home-group-header')){
                const totalLength = item.querySelectorAll('.home-group-header').length;
                title = item.querySelectorAll('.home-group-header')[totalLength - 1].innerHTML;
            }

            masterArray.push({tweets, tweetviews, profileViews, mentions, followers, followPercentage, topFollower, avgTweetPerDay, title })
        } 
        // to view data in table format
        console.log(console.table(masterArray))
        
        // to view data as a JSON String
        console.log(JSON.stringify(masterArray))
    };
    
    
// 5. Hit Enter

// 6. type in console => dataMiner()

// 7. Hit Enter

// 8. You will see output in console as a JSON object, copy that entire object and paste it into https://json-csv.com/ 
// and convert it csv if you ever want it

// 9. Import this CSV to airtable if you want to save it further