Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Star 5 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save atorralb/1d1d5e9333427fa6de0675a2a5c8f371 to your computer and use it in GitHub Desktop.
Save atorralb/1d1d5e9333427fa6de0675a2a5c8f371 to your computer and use it in GitHub Desktop.
iterate through a pagination site with nightmarejs
var Nightmare = require('nightmare');
var vo = require('vo');
vo(run)(function(err, result) {
if (err) throw err;
});
function* run() {
var nightmare = Nightmare(),
MAX_PAGE = 10,
currentPage = 0,
nextExists = true,
links = [];
yield nightmare
.goto('https://www.yahoo.com')
.type('.input-query', 'github nightmare')
.click('#search-submit')
.wait('body')
nextExists = yield nightmare.visible('.next');
while (nextExists && currentPage < MAX_PAGE) {
links.push(yield nightmare
.evaluate(function() {
var links = document.querySelectorAll("ol.searchCenterMiddle a");
return links[0].href;
}));
yield nightmare
.click('.next')
.wait('body')
currentPage++;
nextExists = yield nightmare.visible('.next');
}
console.dir(links);
yield nightmare.end();
}
@dshaw002
Copy link

The examples really outdated and taken from a Github error a year ago. I made some adjustments and it works now:

var Nightmare = require('nightmare');
var vo = require('vo');
vo(run)(function(err, result) {
if (err) throw err;
});

function* run() {
var nightmare = Nightmare(),
MAX_PAGE = 10,
currentPage = 0,
nextExists = true,
links = [];

yield nightmare
    .goto('https://www.yahoo.com')
    .type('#uh-search-box', 'github nightmare')
    .click('#uh-search-button')
    .wait('ol.searchCenterMiddle')


nextExists = yield nightmare.visible('.next');

while (nextExists && currentPage < MAX_PAGE) {
    links.push(yield nightmare
        .evaluate(function() {
            var links = document.querySelectorAll("ol.searchCenterMiddle a");
            console.log(links[0].href);
            return links[0].href;
        }));

        yield nightmare
            .click('.next')
            .wait('body')

        currentPage++;
        nextExists = yield nightmare.visible('.next');
}
console.dir(links);
yield nightmare.end();

}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment