Last active
August 29, 2015 14:21
-
-
Save seangibat/d5011c9547ccf405f68c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var cheerio = require('cheerio'); | |
var request = require('request'); | |
var _ = require('underscore'); | |
var getNumPages = function(id){ | |
return getThreadPage(id).then(function(page){ | |
var $ = cheerio.load(page); | |
var numPages = $('.topic-actions .pagination span a').last().text(); | |
numPages = numPages ? parseInt(numPages) : 1; | |
return numPages; | |
}); | |
}; | |
var numPagesToPageNumbers = function(numPages){ | |
return _.range(1, numPages + 1); | |
}; | |
var getThreadPage = function(id, page){ | |
return new Promise(function(resolve, reject){ | |
page = page || 1; | |
page = (page - 1) * 30; | |
request('http://forums.hipinion.com/viewtopic.php?f=1&t=' + id + '&start=' + page, function(error, response, body){ | |
if (!error && response.statusCode == 200) { | |
resolve(body); | |
} else { | |
reject(error); | |
} | |
}); | |
}); | |
}; | |
var threadPageToYoutubeIds = function(page){ | |
var $ = cheerio.load(page); | |
var urls = $('embed').map(function(i, d){ | |
var url = $(d).attr('src'); | |
return url.split('youtube.com/v/')[1].substring(0,11) | |
}); | |
return urls.get(); | |
}; | |
var getWholeThread = function(id){ | |
return getNumPages(id) | |
.then(numPagesToPageNumbers) | |
.then(function(pageNumbers){ | |
return pageNumbers.map(getThreadPage.bind(null, id)) | |
}) | |
.then(function(requests){ | |
return Promise.all(requests); | |
}) | |
.then(function(pagesArray){ | |
return pagesArray.map(threadPageToYoutubeIds) | |
}) | |
.then(_.flatten); | |
}; | |
module.exports = getWholeThread; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I think maybe you should eliminate the extra then and wrap line 45 in the Promise.all, and then chain a catch to the end so you don't swallow connectivity issues?