Scrape Stackshare Tools to JSON

scrape-stackshare-tools.js

const fetch = require("node-fetch");
const $ = require("cheerio");
const fs = require("fs");

const normalizeVote = (x) => {
    if (/K$/i.test(x)) {
      return +(x.replace(/K/i, '')) * 10**3
    }
    else if (/M$/i.test(x)) {
      return +(x.replace(/M/i, '')) * 10**6
    }  else {
      return x
    }
}

const fetchPage = async page => {
  const url = `https://stackshare.io/tools/top?page=${page}`;
  const result = await fetch(url);
  const html = await result.text();
  const parsedHTML = $.load(html);
  let results = [];

  parsedHTML(".trending-wrapper").map(function(i, el) {
    //
    const $el = $(el);
    // the foo html element into a cheerio object (same pattern as jQuery)
    const title = $el
      .find(".tool-card-title")
      .text()
      .trim();
    const logo = $el.find("img").attr("src");
    const votes = $el.find(".trending-reason-count").toArray().map((x) => $(x).text().trim())
    .map(normalizeVote)
   
    const description = $el
      .find(".trending-description")
      .text()
      .trim();
    const category = $el
      .find("[itemprop=applicationSubCategory]")
      .text()
      .trim();
    const href = $el.find(".tool-card-title a").attr("href");
    const tool = { title, href, description, category, logo, votes: votes[0], stacks: votes[1] };
    // console.log(Object.values(tool).join("|"));
    // console.log(tool);
    results.push(tool);
    return tool;
  });
  // console.log(results);
  return results;
};

const fetchTools = async () => {
  const pages = 1000;
  let page = 1;
  let results = [];

  do {
    console.log(page);
    const pageResults = await fetchPage(page);
    // console.log(pageResults);
    if (pageResults.length === 0 ) break; // reached end
    results.push(...pageResults);
    page++;
  } while (page < pages);

  const json = JSON.stringify(results);
  fs.writeFile("tools.json", json, function(err) {
    if (err) {
      console.log(err);
    }
  });
};

if (require.main === module) {
  fetchTools();
} else {
}
module.exports = {normalizeVote}