MediumScaper.js

const fs = require('fs');
const fetch = require('node-fetch');
const data = require('./data');

const target = 'name="description" content="';
const target2 = 'rel="canonical" href="';

const fetchData = (post) => {
  return fetch(post.url)
    .then(res => res.text())
    .then(res => ({
      indexOfDescription: res.indexOf(target),
      indexOfCanonical: res.indexOf(target2),
      data: res,
    }))
    .then(({ data, indexOfDescription, indexOfCanonical }) => {
      const str = data.slice(indexOfDescription);
      const end = str.indexOf(">");

      const str2 = data.slice(indexOfCanonical);
      const end2 = str2.indexOf(">")

      return {
        ...post,
        description: str.slice(0 + target.length, end -1),
        url: str2.slice(0 + target2.length, end2 -1),
      };
    })
    .catch(err => console.log(`ERROR: ${post.url}`, err))
}

// fetchData(data.posts[0]).then(res => console.log(res))

const augmentData = async (data) => {
  const promises = data.posts.map(post => fetchData(post));
  const results = await Promise.all(promises);
  fs.writeFile('augmented-data.json', JSON.stringify(results, null, 2), () => {
    console.log('complete!')
  })
}

augmentData(data);