Skip to content

Instantly share code, notes, and snippets.

@quanengineering
Last active March 15, 2016 04:01
Show Gist options
  • Save quanengineering/616bbc5f7de87a68d648 to your computer and use it in GitHub Desktop.
Save quanengineering/616bbc5f7de87a68d648 to your computer and use it in GitHub Desktop.
How to use promise
var Q = require('q');
var request = require('request');
var cheerio = require('cheerio');
function getArticleUrlPerPage(pageNumber, articleUrls) {
var deferred = Q.defer();
var options = {
method: 'GET',
url: 'https://www.getnewsmart.com/?last_time_title=This+week&page=' + pageNumber + '&section=&xhr=true',
};
request(options, function(error, response, body) {
if (error) throw new Error(error);
$ = cheerio.load(body);
$('.text-wrapper').each(function(i, elem) {
articleUrls.push($(this).find('.pjax-link').attr('href'));
});
deferred.resolve();
});
return deferred.promise;
}
var articleUrls = [];
var promises = [];
function getArticleUrlWholeSite() {
console.log('Starting to get article urls');
for (var pageNumber = 1; pageNumber <= 64; pageNumber++) {
promises.push(getArticleUrlPerPage(pageNumber, articleUrls)
.then(function() {
console.log(articleUrls.length);
}));
}
Q.allSettled(promises).then(function() {
console.log('End getArticleUrlWholeSite');
});
}
getArticleUrlWholeSite();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment