Incremental deletes against Firebase

incremental-delete.js

// node.js script to perform incremental deletes of a large location in Firebase
//
// USAGE:
// node incremental-delete.js url [batch_size]

var https = require('https');
var url = require('url');

var http_error = function(options, res) {
  return new Error(options['method'] + ' ' + options['path'] + ' returned ' + res.statusCode);
}

var fetch_keys = function(opt, cb) {
  var path = opt.path + '?shallow=true';
  var auth_path = path + (opt.auth ? '&auth=' + opt.auth : '');

  var req = {
    method: 'GET',
    host: opt.host,
    port: 443,
    path: auth_path
  };

  console.log('GET ' + path);
  https.request(req, function(res) {
    var body = '';
    if (res.statusCode == 200) {
      res.on('data', function(d) {
        body += d;
      });
      res.on('end', function() {
        var keys = [];
        if (body !== 'null') {
          keys = Object.keys(JSON.parse(body));
        }
        console.log('Found ' + keys.length + ' keys');
        cb(null, keys);
      });
    } else {
      cb(http_error(req, res));
    }
  }).end();
};

var split_keys = function(keys, step) {
  var result = [];

  for (var i = 0, length = keys.length; i < length; i += step) {
    result.push(keys.slice(i, i + step));
  }

  return result;
};

var delete_keys = function(opt, batch, cb) {
  var patch = {};
  for (var i = 0; i < batch.length; i++) {
    patch[batch[i]] = null;
  }
  var body = JSON.stringify(patch);
  var auth_path = opt.path + (opt.auth ? '?auth=' + opt.auth : '');

  var req = {
    method: 'PATCH',
    host: opt.host,
    port: 443,
    path: auth_path,
    headers: {
      'Content-Type': 'application/json',
      'Content-Length': body.length
    }
  };

  console.log('PATCH ' + opt.path + ' for ' + batch.length + ' keys');
  var stream = https.request(req, function(res) {
    if (res.statusCode != 200) {
      cb(http_error(req, res));
    } else {
      res.on('data', function(chunk) {
        // ignore
      });
      res.on('end', function() {
        cb(null);
      });
    }
  });

  stream.write(body);
  stream.end();
};

var delete_in_batches = function(root, batch_size, delay_ms, cb) {
  fetch_keys(root, function(err, keys) {
    if (err) {
      cb(err);
    } else {
      var batches = split_keys(keys, batch_size);
      var batch = 0;

      function delete_next(err) {
        if (err) {
          cb(err);
        } else if (batch >= batches.length) {
          cb(null, keys.length, batches.length);
        } else {
          delete_keys(root, batches[batch++], function(err) {
            setTimeout(delete_next, delay_ms, err);
          });
        }
      }

      delete_next(null);
    }
  });
};

var main = function() {
  if (process.argv.length < 3) {
    console.log('USAGE: node prune.js URL [BATCH-SIZE]\n');
    console.log('URL: https://FIREBASE.firebaseio.com/PATH.json?auth=TOKEN');
    process.exit(1);
  }

  var parsed = url.parse(process.argv[2], true);
  var batch_size = 50;
  if (process.argv.length > 3) {
    batch_size = parseInt(process.argv[3]);
  }
  var delay_ms = 100;

  var root = {
    host: parsed.hostname,
    port: 443,
    path: parsed.pathname,
    auth: parsed.query.auth
  };

  delete_in_batches(root, batch_size, delay_ms, function(err, keys, batches) {
    if (err) {
      console.log('ERROR: ' + err);
      process.exit(1);
    } else {
      console.log('Deleted ' + keys + ' keys in ' + batches + ' batches');
      process.exit(0);
    }
  });
}

main();