Baisc proof of principle: scraping academic sites by emulated browsing in Node.js with Phantom, Casper, Spooky and JSDom

journal_scrape.js

#! /usr/bin/exec node

var Spooky = require('spooky');
var dom = require('jsdom').jsdom;
var xpath = require('jsdom-xpath');

// Set up the spooky agent to retrieve URLs
var spooky = new Spooky({
  child: {
    transport: 'http'
  },
  casper: {
    logLevel: 'debug',
    verbose: true
  }
}, function (err) {
  if (err) {
    e = new Error('Failed to initialize SpookyJS');
    e.details = err;
    throw e;
  }

  spooky.start("https://peerj.com/articles/384");

  spooky.then(function() {
    this.emit('pagedownload', this.evaluate(function() {
      return document.all[0].outerHTML;
    }));
  });

  spooky.run();
});

spooky.on('pagedownload', function(html) {
  var doc = new dom(html);
  var doi = xpath("//meta[@name='citation_doi']", doc)[0];
  console.log(doi.content);
});