xkcd.js

/**
 * Script to download all comics from xkcd.
 */

'use strict';

const fs = require('fs')
const cheerio = require('cheerio')
const request = require('request')

const N_PARALLEL = 10

const get = index => new Promise((resolve, reject) => {
  request('http://xkcd.com/' + index + '/', (err, res, body) => {
    if (err || !res || (res.statusCode > 399 && res.statusCode !== 404)) {
      reject(err || ('Something went wrong: ' + (res||{}).statusCode))
    } else {
      if (res.statusCode === 404) resolve(null)
      else {
        let img = cheerio.load(body)('img[src*="//imgs.xkcd"]').attr('src')
        if (img) resolve([index, 'http:' + img])
        else { console.log(body); get(index).then(resolve, reject) }
      }
    }
  })
})

let current = 1
const next = () => {
  let reqs = []

  for (let _ = current; _ < (current + N_PARALLEL); _ ++) {
    reqs.push(get(_))
  }

  current += N_PARALLEL

  Promise.all(reqs).then(pics => {
    console.log('* Got %s-%s', current - N_PARALLEL, current)
    pics.forEach(pic => pic&&request(pic[1]).pipe(fs.createWriteStream('./img/'+pic[0]+'.png')))
    if (pics[pics.length - 1]) setTimeout(next, (current-1) % 100 === 0 ? 1000 : 0)
  }, err => {
    console.log('fuck: %s', err)
    process.exit(-1)
  })
}

next()