Use nodejs to download remote resources in batches

node-download.js

const fs = require('fs')
const path = require('path')
const http = require('http')
const https = require('https')
const Stream = require('stream').Transform

// @example
// const list = require('./downloadurls.json')
// downloadList(list, { onItem: res => console.log('✅：', res.basename) })

/**
 * download resource
 * @param  {object} options
 * @param  {string} options.url 下载链接
 * @param  {string} ?options.dir 保存根路径
 * @param  {string} ?options.filename 保存文件名，默认从文件url中读取
 * @return {Promise<Resource>}
 */
function download(options) {
  return new Promise((resolve, reject) => {
    const url = typeof options === 'string' ? options : options.url
    if (!url) throw new TypeError('argument empty `url`')
    const dirname = options.dir || path.resolve(__dirname)
    const basename = options.filename || path.basename(url)
    const filename = path.resolve(dirname, basename)
    const req = url.toString().indexOf("https") === 0 ? https : http
    req.request(url, function (response) {
      const file = fs.createWriteStream(filename)
      response.pipe(file)
      file.on('finish', function () {
        file.close()
        resolve({ url, basename, filename })
      })
      file.on('error', function (err) {
        fs.unlink(filename)
        reject(err)
      })
    }).end()
  })
}

function downloadList(list, options = {}) {
  const { maxTask = 10, onItem = () => {} } = options
  return new Promise((resolve, reject) => {
    const pendingList = [...list]
    const doneList = []
    const taskList = []
    const watchTask = task => task.then(ret => {
      doneList.push(ret)
      onItem(ret)
    }, err => {
      doneList.push(err)
      onItem(err)
    }).finally(() => {
      const taskId = taskList.indexOf(task)
      if (taskId !== -1) {
        taskList.splice(taskId, 1)
      }
      exec()
    })
    const exec = () => {
      if (!pendingList.length) {
        return resolve(doneList)
      }
      if (taskList.length < maxTask) {
        const next = pendingList.shift()
        const task = download(next)
        taskList.push(task)
        watchTask(task)
      }
    }
    return exec()
  })
}


module.exports = {
  download,
  downloadList,
}