Stripe CTF Level 3 Solution

level3.js

var express = require('express')
  , app = express()
  , fs = require('fs')
  , async = require('async')
  , path = require('path')
  , request = require('request')
  , HashTable = require('hashtable')

var port
var indexing = false
var master
var slaves = [1,2,3].map(function (i) { return 'http://localhost:' + (9090 + i)})

process.on('uncaughtException', function (err) {
  console.error('UNCAUGHT', err)
  indexing = false
})

app.use(express.query())

app.get('/index', function (req, res) {
  indexing = true
  if (master) {
    masterIndexDir(req.query.path, function (err) {
      if (err) throw err;
      console.log('all indexing done')
    })
  } else {
    indexAny(path.dirname(req.query.path), req.query.path, function (err) {
      if (err) throw err
      // console.log('index', index.index)
      idx.finalize()
      indexing = false;
      console.log('indexing of ' + req.query.path + ' complete')
    })
    console.log('indexing', req.query.path)
  }
  res.send({success: 'true'})
})

app.get('/healthcheck', function (req, res) {
  if (master) {
    requestSlaves('/healthcheck', function (err, returns) {
      if (err) throw err
      returns.forEach(function (ret) {
        if (ret.resp.statusCode !== 200) throw new Error('non 200 status code')
      })
      res.send({success: 'true'})
    })
  } else {
    console.log('healthcheck', port)
    res.send({success: 'true'})
  }
})

app.get('/isIndexed', function (req, res) {
  if (master) {
    requestSlaves('/isIndexed', function (err, returns) {
      if (err) throw err
      var indexed = !indexing
      returns.forEach(function (ret) {
        indexed = indexed && ret.resp.statusCode === 200;
      })
      if (indexed) {
        // prime up the slaves
        requestSlaves('/?q=pharyngoceratosis', function () {
          res.send({success: true})
        })
      } else {
        res.send(412, 'no')
      }
    })

  } else {
    if (indexing) res.send(412, 'no')
    else res.send({success: true})
  }
})

app.get('/', function (req, res) {
  if (master) {
    var results = []
    requestSlaves('/?q=' + req.query.q, function (err, returns) {
      returns.forEach(function (ret) {
        results.push.apply(results, JSON.parse(ret.body).results)
      })

      if (results.length === 0) {
        console.log('missing result:',  req.query.q)
      }

      res.send({
        success: true,
        results: results
      })
    })
    results.push.apply(results, idx.lookup(req.query.q))
  } else {
    res.send({
      success: true,
      results: idx.lookup(req.query.q)
    })
  }
})

function requestSlaves(path, callback) {
  async.map(slaves, function (slave, cb) {
    request(slave + path, function (err, resp, body) {
      cb(err, {resp: resp, body: body})
    })
  }, callback)
}

function masterIndexDir(pth, callback) {
  async.waterfall([
    fs.readdir.bind(fs, pth),
    function (files, cb) {
      var n = files.length
        , n1 = Math.floor(n/4)
        , n2 = Math.floor(2*n/4)
        , n3 = Math.floor(3*n/4)
      var f1 = files.slice(0, n1)
        , f2 = files.slice(n1, n2)
        , f3 = files.slice(n2, n3)
        , f4 = files.slice(n3)
        , fs = [f1,f2,f3]

      async.map([0,1,2], function (i, cb2) {
        var fl = fs[i]
          , slave = slaves[i]

        console.log('indexing ' + fl + ' at ' + slave)

        async.map(fl, function (f, cb3) {
          request(slave + '/index?path=' + path.join(pth, f), cb3)
        }, cb2)
      }, cb)

      console.log('indexing ' + f4 + ' at master')
      f4.forEach(function (f) {
        var file = path.join(pth, f)
        console.log('indexing', file)
        indexAny(pth, file, function () {
          idx.finalize()
          indexing = false
          console.log('indexing of', file, 'complete')
        })
      })
    }
  ])
}

function indexAny(base, pth, callback) {
  fs.stat(pth, function (err, stat) {
    if (err) throw err
    if (stat.isFile()) {
      indexFile(base, pth, callback)
    } else if (stat.isDirectory()) {
      indexDir(base, pth, callback)
    } else {
      callback()
    }
  })
}

function indexDir(base, pth, callback) {
  async.waterfall([
    fs.readdir.bind(fs, pth),
    function (files, cb) {
      async.map(files, function (file, cb) {
        file = path.join(pth, file)
        fs.stat(file, function (err, stat) {
          if (err) throw err
          indexAny(base, file, cb)
        })
      }, cb)
    }
  ], callback)
}

function indexFile(base, pth, callback) {
  // console.log('indexing', pth)
  async.waterfall([
    fs.readFile.bind(fs, pth),
    function (buf, cb) {
      cb(null, buf.toString())
    },
    indexData.bind(null, pth.replace(base + '/', ''))
  ], callback)
}

function indexData(filename, data, cb) {
  var fi = idx.addFile(filename,data)
  cb()
}

fs.readFile("dictionary.txt", function (err, dict) {
  if (err) throw err
  idx = new Index(dict.toString())
})

// var idx = new Index()
function Index(dict) {
  this.files = []
  this.dict = dict.split('\n')
  this.dict.sort(function (a,b) { return a.length - b.length })
  this.lengthspots = findlengthspots(this.dict)
  this.index = new HashTable()
  this.index.reserve(this.dict.length)
}

Index.prototype.addFile = function (file, data) {
  this.files.push(file)
  var fi = this.files.length - 1
    , I = this

  data.split('\n').forEach(function (line, li) {
    line.split(/[ \.]+/).forEach(function (word, wi) {
      I.add(word, fi, li+1, wi)
    })
  })
}

Index.prototype.subadd = function (word) {
  if (!word) return
  var arr = this.index.get(word)
  if (!arr) arr = []
  arr.push(Array.prototype.slice.call(arguments, 1))
  this.index.put(word, arr)
}

Index.prototype.add = Index.prototype.subadd

Index.prototype.finalize = function () {}

Index.prototype.sublookup = function (word) {
  var I = this
  var result = {}
  var lookup = I.index.get(word)
  lookup && lookup.forEach(function (pair) {
    result[I.files[pair[0]] + ':' + pair[1]] = true
  })
  return Object.keys(result)
}

Index.prototype.lookup = function (word) {
  var results = {}
    , rgx = new RegExp(word)
    , I = this
  this.dict.slice(this.lengthspots[word.length]).forEach(function (dword) {
    if (rgx.test(dword)) {
      I.sublookup(dword).forEach(function (res) {
        results[res] = true
      })
    }
  })
  return Object.keys(results)
}

if (!process.argv[2] || process.argv[2] === '--master') {
  port = 9090
  master = true
} else if (process.argv[2].slice(0,4) === '--id') {
  port = 9090 + parseInt(process.argv[2].split(' ')[1])
  master = false
} else {
  throw new Error('wtf argv ' + process.argv.join(' '))
}
app.listen(port, function () {
  console.log('listening on ' + port)
})

function findlengthspots(d) {
  var spots = {}
  d.forEach(function (word, i) {
    var l = word.length
    if (spots[l]) return
    spots[l] = i
  })
  return spots
}