For details: https://medium.com/p/e6975491d8c0

medium_followers_spider.js

var followers = [];
var totalFollowers = getTotalFollowers()

console.log(`You have ${totalFollowers} follower, crawler start now.`)

var timer = setInterval(() => {
  let liElements = document.querySelectorAll("ul > li");;


  let isEnding = followers.length + liElements.length == totalFollowers;

  if (isEnding) {
    for (let index = 0; index < liElements.length; index++) {
      let follower = parseDataFromLiElement(liElements[index]);
      followers.push(follower);
    }

    console.log("-- crawling over --");
    let result = count(followers)
    console.log(count(followers))
    alert(count(followers))

    clearInterval(timer);
    return;
  }

  // scroll down the webpage
  window.scrollTo(0, document.body.scrollHeight);

  if (liElements.length > 80) {
    // parse first 50
    for (let index = 0; index < 50; index++) {
      let follower = parseDataFromLiElement(liElements[index]);
      // console.log('follower: ', follower)
      followers.push(follower);
    }

    console.log(`The crawler have crawled ${followers.length} followers.`)
    console.log(count(followers))

    // delete some element to reduce memory usage
    for (let index = 0; index < 50; index++) {
      liElements[index].remove();
    }
  }
}, 100);


// parse data from li elements
function parseDataFromLiElement(li) {
  return {
    name: li.querySelector("h2").textContent,
    url: li.querySelector("a").href,
    isMembership: li.querySelector("svg") != null,
    description: li.querySelector("p")?.textContent,
  };
}

function getTotalFollowers() {
  let totalFollowersElement = document.querySelector("h2");
  let numberString = totalFollowersElement.innerText.split(" ")[0];
  return parseInt(numberString.split(",").join(""));
}

function count(followers){
  let membersCount = followers.filter((item) => item.isMembership).length;
  let percentage = (membersCount / followers.length).toFixed(4)
  return `total followers: ${followers.length}, total membership: ${membersCount}, percent: ${percentage * 100}%`
}