function_lexer.rb

class FunctionalLexer
  TOKENS = [
    WhitespaceToken,
    DigitsToken,
    WordToken,
    UnknownToken # keep last
  ]

  include Enumerable

  attr_reader :origin
  attr_reader :source
  attr_reader :source_length

  def initialize(origin, source)
    @origin = origin
    @source = source
    @source_length = source.length
  end

  def each(current_line = 1, current_postion = 0, &block)
    return self unless block_given?

    token_klass, token_text = catch :token do
      TOKENS.each do |token_klass|
        token_text = token_klass.get_next_token_text(source_remaining(current_postion))
        throw :token, [token_klass, token_text] if token_text
      end
    end

    new_line = current_line + token_text.scan(/\n/).count
    new_position = current_position + token_text.length
    location = Location.new(origin, new_line, new_position)
    token = token_klass.new(location, token_text)

    block.call(token)

    # passing new_line and new_position to #each allows us to maintain state without mutating ivars
    each(new_line, new_position, &block) unless new_position == source_length

    self
  end

  alias :tokens :to_a

  def source_remaining(position)
    source[position..source_length]
  end
end

objective_lexer.rb

class ObjectiveLexer
  TOKENS = [
    WhitespaceToken,
    DigitsToken,
    WordToken,
    UnknownToken # keep last
  ]

  include Enumerable

  attr_reader :origin
  attr_reader :source
  attr_reader :source_length

  def initialize(origin, source)
    @origin = origin
    @source = source
    @source_length = source.length
    reset!
  end

  def each(&block)
    return self unless block_given?

    token_klass, token_text = catch :token do
      TOKENS.each do |token_klass|
        token_text = token_klass.get_next_token_text(source_remaining)
        throw :token, [token_klass, token_text] if token_text
      end
    end

    @line = @line + token_text.scan(/\n/).count
    @position = @position + token_text.length
    location = Location.new(origin, @line, @position)
    token = token_klass.new(location, token_text)

    block.call(token)

    each(&block) unless @position == source_length

    # since we are mutating ivars, we should be nice and reset ourself to allow #each to be called again
    reset!
    self
  end

  alias :tokens :to_a

  def source_remaining
    source[@position..source_length]
  end

  def reset!
    @line_number = 1
    @position = 0
  end
end