tumblr-image.js

#!/usr/bin/env node

// script to get all image of target account like lilium.php
// process.argv[2] <- account name
// process.argv[3] <- directory
// process.argv[4] <- if 'quick' it will be quick mode

var fs = require('fs');
var util = require('util');
var http = require('http');
var url = require('url');
var spawn = require('child_process').spawn;

var account, path, quick = false;

switch(process.argv.length) {
case 5:
  if(process.argv[4] == 'quick') { quick = true; }
case 4:
  path = process.argv[3];
  account = process.argv[2];
  break;
default:
  console.error('Expects at least 2 parameter.\nUsase\t: tumblr-image.js [Your Account] [Download destination path]"');
  console.error('Example: lilium.php mystia ./download/');
  process.exit(1);
}

path = (path[path.length - 1] != '/')? path + '/' : path;

var baseAPI = 'http://' + account + '.tumblr.com/api/read/json?type=photo';

if (!fs.existsSync(path)) { fs.mkdirSync(path); }
if (!fs.statSync(path).isDirectory()) {
  console.error('ERROR : Save destination is not directory.');
  process.exit(1);
}

// JSON paser for Tumblr
function tumblr_request(baseAPI, callback) {
  http.get(
    url.parse(baseAPI), function(res) {
      var json_string = '';

      res.setEncoding('utf8');
      res.on('data', function(d) { json_string += d; });
      res.on(
        'end', function() {
          // console.log(json_string);
          // console.log(baseAPI);

          callback(JSON.parse(json_string.replace(/var tumblr_api_read = /, '').replace(/};/, '}')));
        });
    })
    .on(
      'error', function(e) {
        console.error("Got error: " + e.message);
        exit(1);
      });
}

var saveFailedCnt = 0, quick_count = 0, post_total = 0;

var
PER_REQUEST = 50, FREQ_SECOND = 1, QUICK_EXIT_NUMBER = 100,
FETCH_ITEM_MAX = 200, WGET_TIMEOUT = 10;

var job_queue = [];
var running_job_number = 0;

process.on(
  'exit', function() {
    var saveSuccusesCnt = post_total - saveFailedCnt;
    console.log(saveSuccusesCnt + ' images saved!!');
    console.log(saveFailedCnt + ' images save failed.');
  });

function load_posts(offset) {
  tumblr_request(
    baseAPI + '&start='+ offset + '&num=' + PER_REQUEST, function(data) { // GET JSON
      for (var j = 0; j < PER_REQUEST; j++) {
        if (offset + j  >= data['posts-total']) { return; }

        var lastSlashMat = data.posts[j]['photo-url-1280'].match(/\/([^\/]+)$/);
        var mat = !lastSlashMat[1].match(/\.[^.]+/)
          ? data.posts[j]['photo-url-75'].match(/\.[^.]+$/)[0] : '';
        var fileName = path + lastSlashMat[1] + mat;

        if(quick && fs.existsSync(fileName) && (++quick_count > QUICK_EXIT_NUMBER)) { processs.exit(); }

        job_queue.push(['-q', '-c', '-T', WGET_TIMEOUT,
                        '-O', fileName, data.posts[j]['photo-url-1280']]);
         console.log('Status : ' + util.format('(%d / %d)', (offset + j + 1), post_total)
         + ' - ' + ((offset + j + 1) * 100 / data['posts-total']) + ' %'
         + ' : ' + fileName + ' from ' + data.posts[j]['photo-url-1280']
         );
      }
    });
}

tumblr_request(
  baseAPI, function(data) {
    post_total = data['posts-total'];

    console.log('Founded {' + post_total + '} images.');
    console.log('Getting start!!');

    for (var i = 0; i < post_total; i += PER_REQUEST) {
      (function(offset) {
         setTimeout(function() { load_posts(offset); }, 1000 * FREQ_SECOND / PER_REQUEST * offset);
       })(i);
    }
  });

function poller() {
  if(running_job_number >= FETCH_ITEM_MAX) { return; }

  while(job_queue.length > 0 && running_job_number < FETCH_ITEM_MAX) {
    try {
      spawn('wget', job_queue.pop())
        .on('exit', function(code, signal) {
              process.stdout.write('+');
              running_job_number--;
              if(code !== 0) saveFailedCnt++;
            });
      running_job_number++;
    } catch(e) {
      console.error(e);
      process.exit(1);
    }
  }

  setTimeout(poller, 1000 * FREQ_SECOND);

  /*
   http.get(
   url.parse(data.posts[j]['photo-url-1280']), function(res) {
   var file = fs.createWriteStream(fileName);
   res.pause();
   res.on('data', function(d) { file.write(d); });
   res.on('error', function(e) { saveFailedCnt++; });
   res.resume();
   });
   */
}

poller();