Scraper for Seeking Alpha

results.csv

scrapeSeekingAlpha.js

//****************************************************************************
//****************************************************************************
//*** Bountify - https://bountify.co/scraper-for-seeking-alpha             ***
//****************************************************************************
//****************************************************************************

//How to Run
//1. Install Node.js and NPM
//2. Install Packages
//Required Packages
//npm install request
//npm install async
//npm install underscore
//npm cheerio
//3. node scrapeSeekingAlpha.js
//This should generate results.csv

var fs = require('fs'),
    request = require('request'),
    async = require('async'),
    _ = require('underscore'),
    cheerio = require('cheerio');

var outputCSV = [];

//******************************************************
// EACH CATGORY URL (Top 100)
// http://seekingalpha.com/leader-board/opinion-leaders
//******************************************************
var urls = [
"http://seekingalpha.com/opinion-leaders/long-ideas",
"http://seekingalpha.com/opinion-leaders/long-ideas/2",
"http://seekingalpha.com/opinion-leaders/long-ideas/3",
"http://seekingalpha.com/opinion-leaders/long-ideas/4",
"http://seekingalpha.com/opinion-leaders/short-ideas",
"http://seekingalpha.com/opinion-leaders/short-ideas/2",
"http://seekingalpha.com/opinion-leaders/short-ideas/3",
"http://seekingalpha.com/opinion-leaders/short-ideas/4",
"http://seekingalpha.com/opinion-leaders/quick-picks-lists",
"http://seekingalpha.com/opinion-leaders/quick-picks-lists/2",
"http://seekingalpha.com/opinion-leaders/quick-picks-lists/3",
"http://seekingalpha.com/opinion-leaders/quick-picks-lists/4",
"http://seekingalpha.com/opinion-leaders/fund-holdings",
"http://seekingalpha.com/opinion-leaders/fund-holdings/2",
"http://seekingalpha.com/opinion-leaders/fund-holdings/3",
"http://seekingalpha.com/opinion-leaders/fund-holdings/4",
"http://seekingalpha.com/opinion-leaders/insider-ownership",
"http://seekingalpha.com/opinion-leaders/insider-ownership/2",
"http://seekingalpha.com/opinion-leaders/insider-ownership/3",
"http://seekingalpha.com/opinion-leaders/insider-ownership/4",
"http://seekingalpha.com/opinion-leaders/ipo-analysis",
"http://seekingalpha.com/opinion-leaders/ipo-analysis/2",
"http://seekingalpha.com/opinion-leaders/ipo-analysis/3",
"http://seekingalpha.com/opinion-leaders/ipo-analysis/4",
"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy",
"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/2",
"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/3",
"http://seekingalpha.com/opinion-leaders/dividend-investing-strategy/4",
"http://seekingalpha.com/opinion-leaders/income-investing-strategy",
"http://seekingalpha.com/opinion-leaders/income-investing-strategy/2",
"http://seekingalpha.com/opinion-leaders/income-investing-strategy/3",
"http://seekingalpha.com/opinion-leaders/income-investing-strategy/4",
"http://seekingalpha.com/opinion-leaders/dividend-ideas",
"http://seekingalpha.com/opinion-leaders/dividend-ideas/2",
"http://seekingalpha.com/opinion-leaders/dividend-ideas/3",
"http://seekingalpha.com/opinion-leaders/dividend-ideas/4",
"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists",
"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/2",
"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/3",
"http://seekingalpha.com/opinion-leaders/dividend-quick-picks-lists/4",
"http://seekingalpha.com/opinion-leaders/bonds",
"http://seekingalpha.com/opinion-leaders/bonds/2",
"http://seekingalpha.com/opinion-leaders/bonds/3",
"http://seekingalpha.com/opinion-leaders/bonds/4",
"http://seekingalpha.com/opinion-leaders/reits",
"http://seekingalpha.com/opinion-leaders/reits/2",
"http://seekingalpha.com/opinion-leaders/reits/3",
"http://seekingalpha.com/opinion-leaders/reits/4",
"http://seekingalpha.com/opinion-leaders/retirement",
"http://seekingalpha.com/opinion-leaders/retirement/2",
"http://seekingalpha.com/opinion-leaders/retirement/3",
"http://seekingalpha.com/opinion-leaders/retirement/4",
"http://seekingalpha.com/opinion-leaders/market-outlook",
"http://seekingalpha.com/opinion-leaders/market-outlook/2",
"http://seekingalpha.com/opinion-leaders/market-outlook/3",
"http://seekingalpha.com/opinion-leaders/market-outlook/4",
"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals",
"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/2",
"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/3",
"http://seekingalpha.com/opinion-leaders/gold-and-precious-metals/4",
"http://seekingalpha.com/opinion-leaders/commodities",
"http://seekingalpha.com/opinion-leaders/commodities/2",
"http://seekingalpha.com/opinion-leaders/commodities/3",
"http://seekingalpha.com/opinion-leaders/commodities/4",
"http://seekingalpha.com/opinion-leaders/economy",
"http://seekingalpha.com/opinion-leaders/economy/2",
"http://seekingalpha.com/opinion-leaders/economy/3",
"http://seekingalpha.com/opinion-leaders/economy/4",
"http://seekingalpha.com/opinion-leaders/forex",
"http://seekingalpha.com/opinion-leaders/forex/2",
"http://seekingalpha.com/opinion-leaders/forex/3",
"http://seekingalpha.com/opinion-leaders/forex/4",
"http://seekingalpha.com/opinion-leaders/real-estate",
"http://seekingalpha.com/opinion-leaders/real-estate/2",
"http://seekingalpha.com/opinion-leaders/real-estate/3",
"http://seekingalpha.com/opinion-leaders/real-estate/4",
"http://seekingalpha.com/opinion-leaders/demographics",
"http://seekingalpha.com/opinion-leaders/demographics/2",
"http://seekingalpha.com/opinion-leaders/demographics/3",
"http://seekingalpha.com/opinion-leaders/demographics/4",
"http://seekingalpha.com/opinion-leaders/basic-materials",
"http://seekingalpha.com/opinion-leaders/basic-materials/2",
"http://seekingalpha.com/opinion-leaders/basic-materials/3",
"http://seekingalpha.com/opinion-leaders/basic-materials/4",
"http://seekingalpha.com/opinion-leaders/conglomerates",
"http://seekingalpha.com/opinion-leaders/conglomerates/2",
"http://seekingalpha.com/opinion-leaders/conglomerates/3",
"http://seekingalpha.com/opinion-leaders/conglomerates/4",
"http://seekingalpha.com/opinion-leaders/consumer-goods",
"http://seekingalpha.com/opinion-leaders/consumer-goods/2",
"http://seekingalpha.com/opinion-leaders/consumer-goods/3",
"http://seekingalpha.com/opinion-leaders/consumer-goods/4",
"http://seekingalpha.com/opinion-leaders/financial",
"http://seekingalpha.com/opinion-leaders/financial/2",
"http://seekingalpha.com/opinion-leaders/financial/3",
"http://seekingalpha.com/opinion-leaders/financial/4",
"http://seekingalpha.com/opinion-leaders/healthcare",
"http://seekingalpha.com/opinion-leaders/healthcare/2",
"http://seekingalpha.com/opinion-leaders/healthcare/3",
"http://seekingalpha.com/opinion-leaders/healthcare/4",
"http://seekingalpha.com/opinion-leaders/industrial-goods",
"http://seekingalpha.com/opinion-leaders/industrial-goods/2",
"http://seekingalpha.com/opinion-leaders/industrial-goods/3",
"http://seekingalpha.com/opinion-leaders/industrial-goods/4",
"http://seekingalpha.com/opinion-leaders/services",
"http://seekingalpha.com/opinion-leaders/services/2",
"http://seekingalpha.com/opinion-leaders/services/3",
"http://seekingalpha.com/opinion-leaders/services/4",
"http://seekingalpha.com/opinion-leaders/technology",
"http://seekingalpha.com/opinion-leaders/technology/2",
"http://seekingalpha.com/opinion-leaders/technology/3",
"http://seekingalpha.com/opinion-leaders/technology/4",
"http://seekingalpha.com/opinion-leaders/utilities",
"http://seekingalpha.com/opinion-leaders/utilities/2",
"http://seekingalpha.com/opinion-leaders/utilities/3",
"http://seekingalpha.com/opinion-leaders/utilities/4",
"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation",
"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/2",
"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/3",
"http://seekingalpha.com/opinion-leaders/portfolio-strategy-asset-allocation/4",
"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas",
"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/2",
"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/3",
"http://seekingalpha.com/opinion-leaders/etf-long-short-ideas/4",
"http://seekingalpha.com/opinion-leaders/etf-analysis",
"http://seekingalpha.com/opinion-leaders/etf-analysis/2",
"http://seekingalpha.com/opinion-leaders/etf-analysis/3",
"http://seekingalpha.com/opinion-leaders/etf-analysis/4",
"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists",
"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/2",
"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/3",
"http://seekingalpha.com/opinion-leaders/etf-quick-picks-and-lists/4",
"http://seekingalpha.com/opinion-leaders/closed-end-funds",
"http://seekingalpha.com/opinion-leaders/closed-end-funds/2",
"http://seekingalpha.com/opinion-leaders/closed-end-funds/3",
"http://seekingalpha.com/opinion-leaders/closed-end-funds/4",
"http://seekingalpha.com/opinion-leaders/mutual-funds",
"http://seekingalpha.com/opinion-leaders/mutual-funds/2",
"http://seekingalpha.com/opinion-leaders/mutual-funds/3",
"http://seekingalpha.com/opinion-leaders/mutual-funds/4"
]
var authors = [];

async.forEach(urls,function(url,urlcb){
  request(url, function(error, response, body) {
      if (!error && response.statusCode == 200) {
        var $ = cheerio.load(body);
        ////*[@id="author_articles_wrapper"]/div/div[1]/ul
        $('.ld_top_list li a').each(function(i, elm) {          
          authors.push($(this).attr('href'));
        }); 
        $('.ld_more_list li a').each(function(i, elm) {          
          authors.push($(this).attr('href'));
        }); 
        console.log("FETCH: "+ url);
        urlcb();
      }else{
        console.log("Check Connection: Issue viewing: "+ url);
        console.log(error);
        process.exit(0);
      }
  });
},function(){
  var uniqAuthors = _.uniq(authors);

  //@todo - If you want to output each author to text file
  // var wstream = fs.createWriteStream('authors.txt',{'flags': 'a'});
  // async.forEach(uniqAuthors,function(a,acb){
  //   wstream.write(a+"\n");
  // },function(){
  //   wstream.end();
  //   console.log("DONE");
  //   process.exit(0);
  // });

    async.forEach(uniqAuthors,function(name,pcb){
      var url = 'http://seekingalpha.com' + name;
      request(url, function(error, response, body) {
          if (!error && response.statusCode == 200) {
            console.log("Fetching: "+url);
            var $ = cheerio.load(body);
            var author = $("#author_full_name").text();
            var profile = $('.profile_item_mini_text').text().toLowerCase();
            // console.log(name+"|"+profile);
            var boolHedgeFund = contains(profile, 'hedge fund');
            var boolFormer = contains(profile, 'former');
            var boolEx = contains(profile, 'ex');
            var boolPortfolioManager = contains(profile, 'portfolio manager');

            var count=0;
            var keywords =[];
            if(boolHedgeFund){count++;keywords.push('hedge fund')}
            if(boolFormer){count++;keywords.push('former')}
            if(boolEx){count++;keywords.push('ex')}
            if(boolPortfolioManager){count++;keywords.push('portfolio manager')}

            var capture = {
              author:author,
              url:url,
              keywords:keywords,
              // profile: profile,
              count:count
            }
            if(count>0){
              outputCSV.push(capture);
              pcb();
            }else{
              pcb();
            }
          }else{
            pcb();
          }
      });
    },function(){
      console.log("DONE");
      var wstream = fs.createWriteStream('results.csv',{'flags': 'w'});
      wstream.write("Author,URL,Count,Keywords\n");
      async.forEach(outputCSV,function(a,acb){
        wstream.write(JSON.stringify(a.author)+","+a.url+","+a.count+","+JSON.stringify(a.keywords.join())+"\n");
      },function(){
        wstream.end();
        console.log("DONE");
        process.exit(0);
      });
    })
})

function contains(r,s){ 
  return r.indexOf(s) !== -1; 
}