tokenize-map-function.js

// Map function
function(doc) {
  // Split on unicode whitespace or punctuation.
  // TODO: Is there a unicode-aware \W that I could use?
  // log('mapping ' + doc);
  var tokens = doc.text.split(/[\s\,\.\;\:\'\"\(\)\!\?]+/);
  var words = {};
  for each (var token in tokens) {
    token = token.toLowerCase();
    if (token != "") {
      if (token in words) {
        words[token] += 1
      }
      else {
        words[token] = 1
      }
    }
  }
  emit(doc._id, words);
}