Skip to content

Instantly share code, notes, and snippets.

@checkaayush
Last active June 11, 2016 11:12
Show Gist options
  • Save checkaayush/4228b87431c2caed7aa1962a2366c449 to your computer and use it in GitHub Desktop.
Save checkaayush/4228b87431c2caed7aa1962a2366c449 to your computer and use it in GitHub Desktop.
Node.js script to load .txt files from a set of URLs asynchronously and combine results in order of URLs using default modules (and with non-default modules)
// Task using default Node.js packages
var http = require('http');
var urls = ['http://screeningtest.vdocipher.com/file1.txt',
'http://screeningtest.vdocipher.com/file2.txt',
'http://screeningtest.vdocipher.com/file3.txt',
'http://screeningtest.vdocipher.com/file4.txt',
'http://screeningtest.vdocipher.com/file5.txt',
'http://screeningtest.vdocipher.com/file6.txt',
'http://screeningtest.vdocipher.com/file7.txt',
'http://screeningtest.vdocipher.com/file8.txt',
'http://screeningtest.vdocipher.com/file9.txt',
'http://screeningtest.vdocipher.com/file10.txt'];
var responses = [];
var completedRequests = 0;
/**
* Given a 'url', it returns a JSON object of the form
* {"url": <someUrl>, body: <textFileContentAtUrl>}
*/
function httpGet(url, callback) {
var returnVal = {url: url};
http.get(url, function(response) {
var output = '';
response.on('data', function(chunk){
output += chunk;
});
response.on('end', function(){
returnVal['body'] = output.trim();
callback(returnVal);
});
});
}
function httpHelper(urls, callback) {
for (var index = 0; index < urls.length; index++) {
httpGet(urls[index], function(body) {
completedRequests++;
responses.push(body);
if (completedRequests == urls.length) {
callback(responses);
}
});
}
}
httpHelper(urls, function(responses) {
var outputArray = [];
for (var index in responses) {
var urlIndex = urls.indexOf(responses[index].url);
outputArray.splice(urlIndex, 0, responses[index].body);
}
var outputString = outputArray.join();
console.log(outputString);
});
// Additional Method using non-default Node.js packages: 'async' & 'request'
var async = require('async');
var request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
callback(err, body);
}
);
}
async.map(urls, httpGet, function (err, res){
if (err) return console.log(err);
console.log(res.join());
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment