Simple tokenizer function (for js-tokens and css-tokens)

tokenize.coffee

###
Tokenizes `string` into an array of tokens, using `regex` such as [js-tokens]
or [css-tokens].

[js-tokens]:  https://github.com/lydell/js-tokens
[css-tokens]: https://github.com/lydell/css-tokens
###

tokenize = (string, regex) ->
  regex.lastIndex = 0
  if string is ""
    []
  else
    regex.matchToToken match while match = regex.exec string

tokenize.js

// Generated by CoffeeScript 1.8.0

/*
Tokenizes `string` into an array of tokens, using `regex` such as [js-tokens]
or [css-tokens].

[js-tokens]:  https://github.com/lydell/js-tokens
[css-tokens]: https://github.com/lydell/css-tokens
 */
var tokenize;

tokenize = function(string, regex) {
  var match, _results;
  regex.lastIndex = 0;
  if (string === "") {
    return [];
  } else {
    _results = [];
    while (match = regex.exec(string)) {
      _results.push(regex.matchToToken(match));
    }
    return _results;
  }
};