Simple Lexer for arithmetic expressions

simple-lexer.rb

DIGIT = /[0-9]/
LETTER = /[a-z]/i

class Lexer
  
  def self.lex(input)
    output = []
    it = CharIterator.new(input)
    
    peek = it.next
    while peek
    
      case peek
      when DIGIT
        val = 0
        while peek and DIGIT =~ peek
          val = 10 * val + Integer(peek)
          peek = it.next
        end
        output << Token.new(:int, val)
      when LETTER
        var = ""
        while peek and LETTER =~ peek
          var << peek
          peek = it.next
        end
        output << Token.new(:id, var)
      when '+'
        output << Token.new(:add)
        peek = it.next
      when '-'
        output << Token.new(:sub)
        peek = it.next
      when '*'
        output << Token.new(:mul)
        peek = it.next
      when '/'
        output << Token.new(:div)
        peek = it.next
      else
      peek = it.next
      end
    end
    
    return output
  end
end

class Token
  
  def initialize(name, args=nil)
    @name = name
    @args = args
  end
  
  def to_s
    if @args
      "<%s, %s>" % [@name, @args]
    else
      "<%s>" % @name
    end
  end
  
end

class CharIterator
  
  def initialize(input)
    @str = input
    @pos = -1
  end
  
  def next
    @pos += 1
    @str[@pos].chr if @str[@pos]
  end
  
  def next?
    @pos + 1 < @str.length
  end
end

# examples: [ "1", "2", "12", "1+2", "2-1", "2/1", "1*2", "10+20", "20-10", "20/10", "10*20", "a+b", "b-a", "b/a", "a*b", "ab-ba", "ab/ba", "ab*ba"]