Script to iterate a directory structure and do stuff

walk-recursive-flatten-copy.js

#!/usr/bin/env node
/*
 * General folder iterator plus filtering functions.
 * Copyright (2015) Davide 'Folletto' Casali
 * Released under GPLv2 License.
 *
 * USAGE:
 *   node move-flatten.js jpg ./from/ ./to/
 */


var fs = require('fs');
var path = require('path');
var child_process = require('child_process');

// Debut Settings:
var LIMIT = false; // Set this to a number to limit the number of matches



// ************************************************** RECURSIVE WALKER

function walk(dir, fx, countEvery) {
  var count = 1;
  dir = dir.replace(/\/+$/, '');

  (function walkIterator(dir, fx) {
    fs.readdir(dir, function(err, files) {
      if (err) return false;
      files.forEach(function(file) {
        var fullPath = dir + '/' + file;
        //console.log(fullPath);
        fs.stat(fullPath, function(err, stat) {

          if (stat && stat.isDirectory()) {
            // ****** Descent
            walkIterator(fullPath, fx);
          } else {
            // ****** On File Callback
            var didMatch = fx({
              count: count,
              dir: dir,
              file: file
            });

            // Counter
            if (LIMIT) if (count == LIMIT) process.exit(); // DEBUG
            if (didMatch) {
              if (!(count % countEvery)) console.log(count + '...');
              count++;
            }
          }

        });
      });
    });
  })(dir, fx);

}

function walkSync(dir, fx, countEvery) {
  var count = 1;
  dir = dir.replace(/\/+$/, '');

  (function walkIterator(dir, fx) {
    var files = fs.readdirSync(dir);
    if (files) {
      files.forEach(function(file) {
        var fullPath = dir + '/' + file;
        //console.log(fullPath);
        var stat = fs.statSync(fullPath);
        if (stat && stat.isDirectory()) {
          // ****** Descent
          walkIterator(fullPath, fx);
        } else {
          // ****** On File Callback
          var didMatch = fx({
            count: count,
            dir: dir,
            file: file
          });

          // Counter
          if (LIMIT) if (count == LIMIT) process.exit(); // DEBUG
          if (didMatch) {
            if (!(count % countEvery)) console.log(count + '...');
            count++;
          }
        }
      });
    }
  })(dir, fx);

}


// ************************************************** ON FILE FUNCTIONS

function Factory_OnFile_PrintPath(options) {
  return function OnFile_PrintPath(item) {
    console.log('[' + item.count + '] ' + item.dir + '/' + item.file);
    return true;
  }
}

function Factory_OnFile_MatchAndPrintPath(options) {
  return function OnFile_MatchAndPrintPath(item) {
    var fullPath = item.dir + '/' + item.file;
    if (path.extname(fullPath) == '.' + options.matchExt) {
      console.log('[' + item.count + '] ' + fullPath);
      return true;
    }
  }
}

function Factory_OnFile_PrintUniqueExtensions(options) {
  var extensions = {};

  process.on('exit', function() {
    Object.keys(extensions).forEach(function(key) {
      console.log('"' + key + '" = ' + extensions[key]);
    });
  });

  return function OnFile_PrintUniqueExtensions(item) {
    var ext = path.extname(item.file);
    if (!extensions[ext]) {
      extensions[ext] = 1;
      //console.log(ext);
    } else {
      extensions[ext]++;
    }
    return true;
  }
}

function Factory_OnFile_MatchAndCopySafe(options) {
  return function OnFile_MatchAndCopySafe(item) {
    var fullPath = item.dir + '/' + item.file;
    if (path.extname(fullPath) == '.' + options.matchExt) {
      fileCopySafeRetry(fullPath, options.destination + '/' + item.file);
      return true;
    }
  }
}



// ************************************************** SUPPORT FUNCTIONS

function fileCopySafeRetry(from, to) {
  if (!fileCopySafeRetry.counterPerName) fileCopySafeRetry.counterPerName = {};

  (function copyOrRetryIncrement(from, to) {
    try {
      copyFileSyncSafe(from, to);
    } catch(e) {
      if (e.code == 'EEXIST') {
        // File exists, increment and retry
        if (!fileCopySafeRetry.counterPerName[to]) fileCopySafeRetry.counterPerName[to] = 0;
        fileCopySafeRetry.counterPerName[to]++;
        copyOrRetryIncrement(from, pathFilenameSuffix(to, '-' + fileCopySafeRetry.counterPerName[to]));
      } else {
        console.log(e.stack);
        process.exit(2);
      }
    }
  })(from, to);
}

function pathFilenameSuffix(filePath, suffix) {
  return path.dirname(filePath) + '/' + path.basename(filePath, path.extname(filePath)) + suffix + path.extname(filePath);
}


// Modified from 'node-fs-extra': https://github.com/jprichardson/node-fs-extra/blob/master/lib/copy.js
// This function will throw an error if the file if exists ('wx').
function copyFileSyncSafe(srcFile, destFile) {
  var BUF_LENGTH = 64 * 1024
  var _buff = new Buffer(BUF_LENGTH)
  var fdr = fs.openSync(srcFile, 'r')
  var stat = fs.fstatSync(fdr)
  var bytesRead = 1
  var pos = 0

  try {
    var fdw = fs.openSync(destFile, 'wx', stat.mode)
  } catch(e) {
    fs.closeSync(fdr)
    throw(e)
  }

  while (bytesRead > 0) {
    bytesRead = fs.readSync(fdr, _buff, 0, BUF_LENGTH, pos)
    fs.writeSync(fdw, _buff, 0, bytesRead)
    pos += bytesRead
  }

  fs.closeSync(fdr)
  fs.closeSync(fdw)
}


// ************************************************** COMMAND LINE CHECKS

function hasArgumentsOrDie(paramsList) {
  var stumbled = false;
  var out = {};

  Object.keys(paramsList).forEach(function(key, index) {
    if (process && process.argv[index + 2]) {
      out[key] = process.argv[index + 2];
    } else {
      console.log(paramsList[key]);
      stumbled = true;
    }
  });

  if (stumbled) process.exit(1);
  return out;
}


// ************************************************** RUN
var options = hasArgumentsOrDie({
  matchExt: "Missing file extension, i.e.: jpg",
  source: "Missing source folder, i.e.: ./from",
  destination: "Missing destination folder, i.e.: ./to"
});

// Remove trailing slash to be uniform...
if (options.destination) options.destination = options.destination.replace(/\/+$/, '');

// Run
walk(options.source, Factory_OnFile_MatchAndCopySafe(options), 10);
//walk(options.source, Factory_OnFile_PrintUniqueExtensions(options), 1000000);