Functional scraping: A reusable function for extracting data from websites.

data-scraper.js

function extractDataFrom(element) {
  return (options) => {
    let data = {}
    options.forEach(option => {
      const targetElem = element.querySelector(option.selector);
      let targetValue;
      switch (option.attribute) {
        case 'text':
          targetValue = targetElem ? targetElem.innerText.trim() : '';
          break;
        case 'multiline':
        case 'multi':
          targetValue = targetElem ? targetElem.textContent.trim().replace(/\r?\n|\r|\s{2,}/g, ' ') : '';
          break;
        case 'href':
        case 'link':
          targetValue = targetElem ? targetElem.href : '';
          break;
        default:
          targetValue = targetElem ? targetElem.getAttribute(option.attribute) : '';
          break;
      }
      data[option.keyName] = targetValue;
    })
    return data;
  }
}

// Scraping options
const dribbleHomeOpts = [
  {
    keyName: 'thumb',
    selector: '.dribbble-img picture img',
    attribute: 'src',
  },
  {
    keyName: 'creator',
    selector: '.attribution-user .display-name',
    attribute: 'text',
  },
  {
    keyName: 'profilePage',
    selector: '.attribution-user a',
    attribute: 'link',
  },
  {
    keyName: 'favorites',
    selector: '.toggle-fav',
    attribute: 'text',
  },
]

// Example: Scraping multiple posts
Array.from(document.querySelectorAll('#main ol li.shot-thumbnail'))
  .map(shot => extractDataFrom(shot)(dribbleHomeOpts))

Notes to self: Plenty of room for improvement left. Maybe a custom regex and a boolean test option. Method used for cleaning up multiline text could also use some improvements.