Skip to content

Instantly share code, notes, and snippets.

@seangibat
Last active August 29, 2015 14:21
Show Gist options
  • Save seangibat/d5011c9547ccf405f68c to your computer and use it in GitHub Desktop.
Save seangibat/d5011c9547ccf405f68c to your computer and use it in GitHub Desktop.
var cheerio = require('cheerio');
var request = require('request');
var _ = require('underscore');
var getNumPages = function(id){
return getThreadPage(id).then(function(page){
var $ = cheerio.load(page);
var numPages = $('.topic-actions .pagination span a').last().text();
numPages = numPages ? parseInt(numPages) : 1;
return numPages;
});
};
var numPagesToPageNumbers = function(numPages){
return _.range(1, numPages + 1);
};
var getThreadPage = function(id, page){
return new Promise(function(resolve, reject){
page = page || 1;
page = (page - 1) * 30;
request('http://forums.hipinion.com/viewtopic.php?f=1&t=' + id + '&start=' + page, function(error, response, body){
if (!error && response.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
};
var threadPageToYoutubeIds = function(page){
var $ = cheerio.load(page);
var urls = $('embed').map(function(i, d){
var url = $(d).attr('src');
return url.split('youtube.com/v/')[1].substring(0,11)
});
return urls.get();
};
var getWholeThread = function(id){
return getNumPages(id)
.then(numPagesToPageNumbers)
.then(function(pageNumbers){
return pageNumbers.map(getThreadPage.bind(null, id))
})
.then(function(requests){
return Promise.all(requests);
})
.then(function(pagesArray){
return pagesArray.map(threadPageToYoutubeIds)
})
.then(_.flatten);
};
module.exports = getWholeThread;
@spasiu
Copy link

spasiu commented May 26, 2015

I think maybe you should eliminate the extra then and wrap line 45 in the Promise.all, and then chain a catch to the end so you don't swallow connectivity issues?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment