Last active
August 29, 2015 14:12
-
-
Save D1no/155ad8f012dfc06ed991 to your computer and use it in GitHub Desktop.
Cheerio scrape gist with Meteor
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
packages | |
meteorhacks:npm | |
meteorhacks:async | |
node modules | |
request | |
cheerio | |
*/ | |
// Something like Calling a method inside a template to past the result inside a session variable | |
//initiate scraping | |
//encoding should be "binary" to deal with UTF-8 content | |
Meteor.call("scrape", scrapeURL, scrapeSelector, scrapeEncoding, scrapeOutput, function (err, res) { | |
if (err) console.log("Error calling scraper" + err); | |
Session.set("scraperRaw", res); | |
return "Scraped Successfully"; | |
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// in server/ folder | |
Meteor.methods({ | |
'scrape': function scrape(url, cssSelector, encoding, scrapeOutput) { | |
var request = new Meteor.npmRequire('request'); | |
var cheerio = new Meteor.npmRequire('cheerio'); | |
var response = Async.runSync(function(done) { | |
request({uri: url, encoding: encoding}, function(err, resp, body) { | |
if (err) return new Meteor.Error(500 ,"Scraper requires http:// - Protocol as URL", err); | |
if (resp.statusCode != 200) return new Meteor.Error(500 ,"URL responds with " + resp.statusCode, resp); | |
//load desired html section | |
var $ = cheerio.load(body); | |
var selection = $(cssSelector); | |
if (scrapeOutput == "text") { | |
done (null, $(selection).text()); | |
} else { | |
done (null, $(selection).html()); | |
} | |
}); | |
}); | |
return response.result; | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment