Skip to content

Instantly share code, notes, and snippets.

@D1no
Last active August 29, 2015 14:12
Show Gist options
  • Save D1no/155ad8f012dfc06ed991 to your computer and use it in GitHub Desktop.
Save D1no/155ad8f012dfc06ed991 to your computer and use it in GitHub Desktop.
Cheerio scrape gist with Meteor
/*
packages
meteorhacks:npm
meteorhacks:async
node modules
request
cheerio
*/
// Something like Calling a method inside a template to past the result inside a session variable
//initiate scraping
//encoding should be "binary" to deal with UTF-8 content
Meteor.call("scrape", scrapeURL, scrapeSelector, scrapeEncoding, scrapeOutput, function (err, res) {
if (err) console.log("Error calling scraper" + err);
Session.set("scraperRaw", res);
return "Scraped Successfully";
});
// in server/ folder
Meteor.methods({
'scrape': function scrape(url, cssSelector, encoding, scrapeOutput) {
var request = new Meteor.npmRequire('request');
var cheerio = new Meteor.npmRequire('cheerio');
var response = Async.runSync(function(done) {
request({uri: url, encoding: encoding}, function(err, resp, body) {
if (err) return new Meteor.Error(500 ,"Scraper requires http:// - Protocol as URL", err);
if (resp.statusCode != 200) return new Meteor.Error(500 ,"URL responds with " + resp.statusCode, resp);
//load desired html section
var $ = cheerio.load(body);
var selection = $(cssSelector);
if (scrapeOutput == "text") {
done (null, $(selection).text());
} else {
done (null, $(selection).html());
}
});
});
return response.result;
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment