Skip to content

Instantly share code, notes, and snippets.

@LiamKarlMitchell
Last active September 3, 2019 10:34
Show Gist options
  • Save LiamKarlMitchell/027b77b9e80ab518c23dd423963cbcda to your computer and use it in GitHub Desktop.
Save LiamKarlMitchell/027b77b9e80ab518c23dd423963cbcda to your computer and use it in GitHub Desktop.
node.js download file outputs progress and does not re-download already downloaded file.
var fs = require('fs');
var http = require('http');
var forceredownload = false;
function downloadFile(url, file, callback, redirect_count, known_size) {
console.log('Attempting to download ' + url + ' to ' + file);
if (redirect_count) {
if (redirect_count > 5) {
callback('Max redirects reached', url);
return;
}
} else {
redirect_count = 0;
}
var filename = DOWNLOAD_DIR + file;
var outfile;
if (callback === undefined) {
console.log('WARNING downloadFile callback is undefined for ' + url + ' to ' + filename);
callback = function(err, data) {
console.log('downloadFile callback result undefined: ', err, data)
};
}
var request = http.get(url, function(response) {
var content_length = 0;
var downloaded_bytes = 0;
function outputInfo() {
if (content_length === undefined) {
console.log(file + ' Progress: unknown\t' + downloaded_bytes + '/' + '?');
}
percent = parseInt((downloaded_bytes / content_length) * 100);
console.log(file + ' Progress: ' + percent + '%\t' + downloaded_bytes + '/' + content_length);
}
console.log(file + ' ' + response.statusCode);
switch (response.statusCode) {
case 200:
// Try to get the content length.
// If the server does not tell us try to use known_size argument.
content_length = response.headers['content-length'] || known_size;
// Check if file was already downloaded.
// Note: we are just checking length not a checksum of any kind or created date.
fs.stat(filename, function(err, stats) {
if (!err) {
console.log('Have already downloaded ' + filename);
if (stats.size == content_length) {
console.log('And the file size matches (' + content_length + ')');
if (forceredownload === false) {
console.log('Not Redownloading');
callback(null, filename);
request.abort();
return;
} else {
console.log('Redownloading');
}
} else {
console.log('But the file size does not match so re-downloading Old(' + stats.size + ') vs New(' + content_length + ')');
}
}
outfile = fs.createWriteStream(file);
response.pipe(outfile);
response.on('data', function(data) {
downloaded_bytes += data.length;
outputInfo();
});
outfile.on('finish', function() {
console.log('Downloaded ' + url + ' to ' + file);
callback(null, file);
});
});
break;
// Handle redirects.
case 302:
new_remote = response.headers.location;
console.log('Redirect ' + url + ' to ' + new_remote);
if (redirect_count <5) {
downloadFile(new_remote, file, callback, redirect_count++);
} else {
console.error('Redirect limit encountered for '+url+'.');
callback(302, null);
}
// Note: Some servers redirect requests to another url like deny.site.com if there are too many rapidly/failing requests.
// You might want to detect that and delay a redirect to the same location.
return;
break;
case 404:
console.log("File Not Found " + url);
// Note: You might want to break here and call callback with a null if you don't care if the file was unable to be downloaded.
default:
console.log('Problem downloading ' + url + ' STATUS: ' + response.statusCode);
request.abort();
callback(response.statusCode, null);
return;
}
}).on('error', function(e) {
console.log('Error Downloading: ' + url + ' ' + e.message);
callback(e, null);
});
}
@LiamKarlMitchell
Copy link
Author

Just something I made to download files without overwriting the file if it already exists.

@abdolrhman
Copy link

@LiamKarlMitchell is this still working

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment