tokenizer.js

function Tokenizer(input) {
  this.input = input;
  this.length = input.length;
  this.index = 0;
  this.cutStart = 0;
  this.cutEnd = 0;
}

var OPEN_BRACE = '{';
var CLOSE_BRACE = '}';
var COLON = ':';
var COMMA = ',';
var SINGLE_QUOTE = '\'';
var HASH = '#';
var PIPE = '|';
var NUMBER_PATTERN = /[0-9]/;
var LETTER_PATTERN = /[a-zA-Z]/;

Tokenizer.prototype.nextToken = function() {
  var ret;

  if (this.index < this.length) {
    var i = this.index;
    var c = this.input.charAt(i);

    this.cutStart = i;
    this.cutEnd = i;

    if (LETTER_PATTERN.test(c)) {
      this.findCutRange(LETTER_PATTERN);
      ret = makeString(i, this.input.substring(this.cutStart, this.cutEnd));
    } else if (c === OPEN_BRACE) {
      ret = makeOpenBrace(i);
    } else if (c === CLOSE_BRACE) {
      ret = makeCloseBrace(i);
    } else if (c === COLON) {
      ret = makeColon(i);
    } else if (c === COMMA) {
      ret = makeComma(i);
    } else if (c === HASH) {
      ret = makeHash(i);
    } else if (c === PIPE) {
      ret = makePipe(i);
    } else if (NUMBER_PATTERN.test(c)) {
      this.findCutRange(NUMBER_PATTERN);
      ret = makeNumber(i, this.input.substring(this.cutStart, this.cutEnd));
    } else {
      ret = makeUnknown(i, c);
    }
  }

  this.index++;

  return ret;
};

Tokenizer.prototype.findCutRange = function(pattern) {
  var c;

  do {
    this.index++;
    this.cutEnd++;
    c = this.input.charAt(this.index);
  } while(pattern.test(c));

  this.index--;
};

function toTokens(input) {
  var tokenizer = new Tokenizer(input);
  var token = tokenizer.nextToken();
  var allTokens = [];

  while(token) {
    allTokens.push(token);
    token = tokenizer.nextToken();
  }

  return allTokens;
}

function makeUnknown(index, data) {
  return {
    type: 'UNKNOWN',
    index: index,
    data: data
  };
}

function makeString(index, data) {
  return {
    type: 'STRING',
    index: index,
    data: data
  };
}

function makeNumber(index, data) {
  return {
    type: 'NUMBER',
    index: index,
    data: data
  };
}

function makeOpenBrace(index) {
  return {
    type: 'OPEN_BRACE',
    index: index
  };
}

function makeCloseBrace(index) {
  return {
    type: 'CLOSE_BRACE',
    index: index
  };
}

function makeColon(index) {
  return {
    type: 'COLON',
    index: index
  };
}

function makeComma(index) {
  return {
    type: 'COMMA',
    index: index
  };
}

function makeHash(index) {
  return {
    type: 'HASH',
    index: index
  };
}

function makePipe(index) {
  return {
    type: 'PIPE',
    index: index
  };
}