-
-
Save atorralb/1d1d5e9333427fa6de0675a2a5c8f371 to your computer and use it in GitHub Desktop.
| var Nightmare = require('nightmare'); | |
| var vo = require('vo'); | |
| vo(run)(function(err, result) { | |
| if (err) throw err; | |
| }); | |
| function* run() { | |
| var nightmare = Nightmare(), | |
| MAX_PAGE = 10, | |
| currentPage = 0, | |
| nextExists = true, | |
| links = []; | |
| yield nightmare | |
| .goto('https://www.yahoo.com') | |
| .type('.input-query', 'github nightmare') | |
| .click('#search-submit') | |
| .wait('body') | |
| nextExists = yield nightmare.visible('.next'); | |
| while (nextExists && currentPage < MAX_PAGE) { | |
| links.push(yield nightmare | |
| .evaluate(function() { | |
| var links = document.querySelectorAll("ol.searchCenterMiddle a"); | |
| return links[0].href; | |
| })); | |
| yield nightmare | |
| .click('.next') | |
| .wait('body') | |
| currentPage++; | |
| nextExists = yield nightmare.visible('.next'); | |
| } | |
| console.dir(links); | |
| yield nightmare.end(); | |
| } |
This code does not work. On my Ubuntu 16.04 box with the latest stable version of node, I got a series of errors, starting with:
/home/dev/node_modules/vo/lib/wrap.js:30
return co(fn).apply(ctx, args.concat(next))
^
TypeError: co(...).apply is not a function
at wrap (/home/dev/node_modules/vo/lib/wrap.js:30:21)
at func (/home/dev/node_modules/vo/lib/compile.js:44:23)
at next (/home/dev/node_modules/vo/lib/pipeline.js:34:15)
at Pipeline (/home/dev/node_modules/vo/lib/pipeline.js:47:8)
at /home/dev/node_modules/vo/index.js:44:5
at vo (/home/dev/node_modules/vo/index.js:97:7)
at Object.<anonymous> (/home/dev/AUTOSIFT/ez_equity_daemon/scraping/nightmare/nightmare_tests.js:61:8)
at Module._compile (module.js:569:30)
at Object.Module._extensions..js (module.js:580:10)
at Module.load (module.js:503:32)
Note that I am not using the co library. I have explicitly required vo. But for some reason I am seeing co listed in the error.
Just for grins, I npm uninstalled co and vo from my packages. Then reinstalled them. After that, the code would execute, except that I got another error, which seems to reference the first if (err) throw err; as follows:
nightmare_tests.js:62
if (err) { throw err; }
^
TypeError: You may only yield a function, promise, generator, array, or object, but the following was passed: "undefined"
at next (/home/dev/node_modules/vo/node_modules/co/index.js:106:12)
at /home/dev/node_modules/vo/node_modules/co/index.js:93:18
at /home/dev/node_modules/vo/node_modules/co/index.js:224:7
at <anonymous>
After removing the err statement, I run the code and it simply hangs.
The examples really outdated and taken from a Github error a year ago. I made some adjustments and it works now:
var Nightmare = require('nightmare');
var vo = require('vo');
vo(run)(function(err, result) {
if (err) throw err;
});
function* run() {
var nightmare = Nightmare(),
MAX_PAGE = 10,
currentPage = 0,
nextExists = true,
links = [];
yield nightmare
.goto('https://www.yahoo.com')
.type('#uh-search-box', 'github nightmare')
.click('#uh-search-button')
.wait('ol.searchCenterMiddle')
nextExists = yield nightmare.visible('.next');
while (nextExists && currentPage < MAX_PAGE) {
links.push(yield nightmare
.evaluate(function() {
var links = document.querySelectorAll("ol.searchCenterMiddle a");
console.log(links[0].href);
return links[0].href;
}));
yield nightmare
.click('.next')
.wait('body')
currentPage++;
nextExists = yield nightmare.visible('.next');
}
console.dir(links);
yield nightmare.end();
}
Nicely done. I'm new to generators. Why do you need the last
yieldstatement? I noticed the code breaks without it, but can't understand why.