Migration Stuffs

find_formats.js

// Still run as sudo
var fs = require("fs");
var path = require("path");

var defaultFormats = [];

defaultFormats.push(/^(.+) (.+):([A-Fa-f0-9]+)->(.+) ([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}) #([0-9-]+) 1 1$/);
defaultFormats.push(/^(.+):(.+)::(([A-Za-z0-9_]|\.|\/|\$)+)$/);
defaultFormats.push(/^(.+):(.+):([0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}):(([A-Za-z0-9_]|\.|\/|\$)+)$/gm);

fs.readdir(".", function(err, files) {
  if (err)
    return console.error(err);
    
  var extraneous = [];
  
  for (var i = 0; i < files.length; i++) {
    var filename = path.join(".", files[i]);
    var stat = fs.statSync(filename);
    if (stat.isFile() && /\.txt$/.test(files[i])) {
      var lines = fs.readFileSync(filename, "utf8").toString().split("\n");
      
      for (var j = 0; j < lines.length; j++) {
        var line = lines[j];
        
        var conforms = false;
        
        for (var k = 0; k < defaultFormats.length; k++) {
          if (defaultFormats[k].test(line))
            conforms = true;
        }
        
        if (!conforms)
          extraneous.push(line);
      }
    }
  }
  
  console.log("Format search completed. The following " + extraneous.length + " lines have an invalid format:\n");
  
  for (var i = 0; i < extraneous.length; i++) {
    console.log(extraneous[i]);
  }
});

occam.js

console.log("Razor.");

var fs = require("fs");
var path = require("path");

fs.readdir(".", function (err, files) {
    if (err)
        return console.error(err);

    for (var i = 0; i < files.length; i++) {
        var dbName = files[i].replace(/\.txt$/g, "");
        var dbFileName = path.join("./export", files[i]);

        var filename = path.join(".", files[i]);
        var stat = fs.statSync(filename);
        if (stat.isFile() && /\.txt$/.test(files[i]) && !/^extraneous\.txt$/.test(files[i])) {
            var store = [];
            var contentWithoutJunk = fs.readFileSync(filename, "utf8").toString()
                .replace(/^[0-9]+/gm, "")
                .replace(/[0-9]+ 1$/gm, "");
            var lines = contentWithoutJunk.split("\n");

            for (var j = 0; j < lines.length; j++) {
                var shouldAdd = true;
                var result = {};
                var line = lines[j];

                // Now break into segments
                var segments = line.split(/:|->| /).filter(function (segment) {
                    return segment !== undefined;
                }).map(function (segment) {
                    return segment.trim();
                }).filter(function (segment) {
                    return segment.length > 1
                        && !/Users\):?/i.test(segment)
                        && !/^Total$/i.test(segment)
                        && segment !== "account_accountsupplier"
                        && !/^([0-9]| )+$/.test(segment)
                        && !/^#[0-9]{4}/.test(segment);
                });

                for (var k = 0; k < segments.length; k++) {
                    var segment = segments[k];
                    if (/^[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}\.[0-9]{0,3}$/.test(segment)) {
                        result.ip = segment;
                    } else if (/^\$H\$/.test(segment)) {
                        result.password = segment;
                    } else if (/^.+@[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]{2,})+$/.test(segment)) {
                        result.email = segment;
                    } else if (/^[a-f0-9]{32}$/i.test(segment) && (segment.length == 32 || segment.length == 49)) {
                        result.password = segment;
                    } else {
                        result.username = segment;
                    }
                }

                if (shouldAdd)
                    store.push(result);
            }

            fs.writeFileSync(decodeURIComponent(dbFileName).replace(/\.txt$/, ".json"), JSON.stringify(store));
        }
    }
});