Lexing Ruby code

lexing.rb

to_lex = <<RUBY
1 + 1
RUBY

# =====  https://rubygems.org/gems/parser  =====
require 'parser/ruby23'
lexer = Parser::Lexer.new(23)
lexer.diagnostics   = Parser::Diagnostic::Engine.new
lexer.static_env    = Parser::StaticEnvironment.new
lexer.source_buffer = Parser::Ruby23.send(:setup_source_buffer, 'file.rb', 1, to_lex, Encoding::UTF_8)
tokens = []
tokens << lexer.advance while tokens.empty? || tokens[-1][0]
tokens
# => [[:tINTEGER, [1, #<Parser::Source::Range file.rb 0...1>]],
#     [:tPLUS, ["+", #<Parser::Source::Range file.rb 2...3>]],
#     [:tINTEGER, [1, #<Parser::Source::Range file.rb 4...5>]],
#     [:tNL, [nil, #<Parser::Source::Range file.rb 5...6>]],
#     [false, ["$eof", #<Parser::Source::Range file.rb 6...6>]]]


# =====  http://www.rubydoc.info/stdlib/core/Ripper  =====
require 'ripper'
Ripper.lex(to_lex)
# => [[[1, 0], :on_int, "1"],
#     [[1, 1], :on_sp, " "],
#     [[1, 2], :on_op, "+"],
#     [[1, 3], :on_sp, " "],
#     [[1, 4], :on_int, "1"],
#     [[1, 5], :on_nl, "\n"]]


# =====  https://rubygems.org/gems/ruby_parser  =====
require 'ruby_parser'
lexer    = RubyLexer.new 23
lexer.ss = RPStringScanner.new to_lex
tokens   = []
tokens << lexer.next_token while tokens.last || tokens.empty?
tokens
# => [[:tINTEGER, 1], [:tPLUS, "+"], [:tINTEGER, 1], [:tNL, nil], nil]