Node.js downloader.js

downloader.js

var fs = require('fs');
var axios = require('axios');
const async = require('async');
const path = require('path');

const DL_INTERVAL_TIMER = 500; //in milliseconds.

module.exports = {
    createFolder: function (dir) {
        if (!fs.existsSync(dir)) {
            fs.mkdirSync(dir, 0766, function (err) {
                console.log(err);
            });
        }
    },
    downloadFiles: function (arr, FOLDER_LOCATION) {
        if (!arr) {
            return;
        }
        if (arr && arr.length < 1) {
            console.log("EMPTY ARRAY ", arr);
            return;
        } else {
            this.createFolder(FOLDER_LOCATION);

            async.eachLimit(arr, 1, (reqUrl, next) => {
                this.downloadFile(reqUrl, FOLDER_LOCATION).then(() => {
                    setTimeout(function () {
                        next();
                    }, DL_INTERVAL_TIMER);
                })
            });
        }
    },
    downloadFile: async function (requestUrl, FOLDER_LOCATION) {
        if (!requestUrl) {
            console.log("url not found");
            return;
        }

        var response = null;
        try {
            response = await axios({
                method: 'GET',
                url: requestUrl,
                responseType: 'stream'
            });
        } catch (ex) {
            return new Promise((resolve, reject) => { resolve() });
        }



        var fileName = this.getFilenameFromHeaders(response) || "default-" + (new Date().getTime()) + ".jpg";
        var fullPath = path.resolve(FOLDER_LOCATION, fileName);
        response.data.pipe(fs.createWriteStream(fullPath));

        return new Promise((resolve, reject) => {
            response.data.on('end', () => {
                console.log('done : ' + requestUrl);
                resolve()
            })

            response.data.on('error', () => {
                reject()
            })
        });
    },
    getFilenameFromHeaders: function (resp) {
        var regexp = /filename=\"(.*)\"/gi;
        var str = resp.headers['content-disposition'];
        if (!str) {
            return resp.config.url.split("/").pop();
        }
        return regexp.exec(resp.headers['content-disposition'])[1];
    }
}