CLTK Lexer for Crystal

lexer.cr

require "./ctype"
require "cltk/streamposition"
require "cltk/token"
require "cltk/scanner"

module Pool
  class Lexer < CLTK::Scanner
    # extend CLTK::Scanner::LexerCompatibility

    # Skip whitespace
    rule("\n")                        { { :CR         } }
    rule(" ")

    # Keywords
    rule("abstract")                  { { :ABSTRACT       } }
    rule("alias")                     { { :ALIAS          } }
    rule("asm")                       { { :ASM            } }
    rule("class")                     { { :CLASS          } }
    rule("def")                       { { :DEF            } }
    rule("end")                       { { :END            } }
    rule("include")                   { { :INCLUDE        } }
    rule("extend")                    { { :EXTEND         } }
    rule("module")                    { { :MODULE         } }
    rule("nil")                       { { :NIL            } }
    rule("private")                   { { :PRIVATE        } }
    rule("protected")                 { { :PROTECTED      } }
    rule("require")                   { { :REQUIRE        } }
    rule("return")                    { { :RETURN         } }
    rule("self")                      { { :SELF           } }
    rule("struct")                    { { :STRUCT         } }
    rule("super")                     { { :SUPER          } }
    rule("type")                      { { :TYPE           } }
    rule("typeof")                    { { :TYPEOF         } }
    rule("uninitialized")             { { :UNINITIALIZED  } }
    rule("annotation")                { { :ANNOTATION     } }
    rule("union")                     { { :UNION          } }

    # Blocks
    rule("do")                        { { :DO             } }
    rule("yield")                     { { :WITH           } }
    rule("with")                      { { :WITH           } }
    rule("->")                        { { :"->"           } }

    # Exception handling
    rule("begin")                     { { :BEGIN          } }
    rule("raise")                     { { :RAISE          } }
    rule("rescue")                    { { :RESCUE         } }

    # Control flow
    rule("if")                        { { :IF             } }
    rule("else")                      { { :ELSE           } }
    rule("unless")                    { { :UNLESS         } }
    rule("elsif")                     { { :ELSIF          } }
    rule("then")                      { { :ELSIF          } }
    rule("case")                      { { :CASE           } }
    rule("next")                      { { :NEXT           } }
    rule("break")                     { { :BREAK          } }
    rule("for")                       { { :FOR            } }

    # Loops
    rule("while")                     { { :WHILE          } }
    rule("loop")                      { { :LOOP           } }
    rule("until")                     { { :UNTIL          } }

    # Arithmetic Operators
    rule("+")	                        { { :"+"            } }
    rule("-")	                        { { :"-"            } }
    rule("\*")	                      { { :"*"            } }
    rule("/")	                        { { :"/"            } }
    rule("%")	                        { { :"%"            } }
    rule("\*\*")	                    { { :"**"           } }

    # Comparison Operators
    rule("==")	                      { { :"=="           } }
    rule("!=")	                      { { :"!="           } }
    rule("<")	                        { { :"<"            } }
    rule(">")	                        { { :">"            } }
    rule(">=")	                      { { :">="           } }
    rule("<=")	                      { { :"<="           } }
    rule("<=>")	                      { { :"<=>"          } }
    rule("===")	                      { { :"==="          } }

    # Assignment Operators
    rule("=")	                        { { :"="            } }
    rule("+=")	                      { { :"+="           } }
    rule("-=")	                      { { :"-="           } }
    rule("\*=")	                      { { :"*="           } }
    rule("/=")	                      { { :"/="           } }
    rule("%=")	                      { { :"%="           } }
    rule("%=")	                      { { :"%="           } }
    rule("\*\*=")                     { { :"**="          } }

    # Bitwise Operators
    rule("&")	                        { { :"&"            } }
    rule("|")	                        { { :"|"            } }
    rule("^")	                        { { :"^"            } }
    rule("~")	                        { { :"~"            } }
    rule("<<")	                      { { :"<<"           } }
    rule(">>")	                      { { :">>"           } }

    # Logical Operators
    rule("&&")                        { { :"&&"           } }
    rule("&&=")                       { { :"&&="          } }
    rule("&+=")                       { { :"&+="          } }
    rule("&-=")                       { { :"&-="          } }
    rule("&**")                       { { :"&**"          } }
    rule("&*=")                       { { :"&*="          } }
    rule("||")	                      { { :"||"           } }
    rule("!")                         { { :"!"            } }

    # Range Operators
    rule("..")	                      { { :".."           } }
    rule("...")	                      { { :"..."          } }

    # Uncategorized Operators
    rule("::")                        { { :"::"           } }
    rule(".")	                        { { :"."            } }

    # Separators
    rule("(")	                        { { :"("            } }
    rule(")")	                        { { :")"            } }
    rule("[")	                        { { :"["            } }
    rule("]")	                        { { :"]"            } }
    rule("@[")                        { { :"@["           } }
    rule("{")	                        { { :"{"            } }
    rule("}")	                        { { :"}"            } }
    rule(":")	                        { { :":"            } }
    rule("?")	                        { { :"?"            } }
    rule(";")	                        { { :";"            } }
    rule(",")	                        { { :","            } }

    # Identifiers
    rule(/".*"/)                      { |t| { :STRING,        t[1..-2]  } }
    rule(/'[^\']{1,1}'/)              { |t| { :CHAR,          t[1]      } }
    rule(/[A-Z][_A-Za-z0-9]*/)        { |t| { :CONST,         t         } }
    rule(/:[\S]*/)                    { |t| { :SYMBOL,        t         } }
    rule(/@[a-z0-9_]*/)               { |t| { :INSTANCE_VAR,  t         } }
    rule(/@@[a-z0-9_]*/)              { |t| { :CLASS_VAR,     t         } }
    rule(/[a-z][_A-Za-z0-9]*/)        { |t| { :IDENT,         t         } }

    # Macros
    rule(/%[iqQrxw]\(.*\)/)           { |t| { :MACRO, t                 } }
    rule(/%[iqQrxw]\[.*\]/)           { |t| { :MACRO, t                 } }
    rule(/%[iqQrxw]\{.*\}/)           { |t| { :MACRO, t                 } }
    rule(/%[iqQrxw]\<.*\>/)           { |t| { :MACRO, t                 } }
    rule(/%[iqQrxw]\|.*\|/)           { |t| { :MACRO, t                 } }

    # Numbers
    rule(/\d+/)                       { |t| { :NUMBER,  t.to_f          } }
    rule(/\.\d+/)                     { |t| { :NUMBER,  t.to_f          } }
    rule(/\d+\.\d+/)                  { |t| { :NUMBER,  t.to_f          } }

    # Comments
    rule(/#\s*/)                      { push_state(:comment)              }
    rule(/[^\n]+/,  :comment)         { |t| { :COMMENT, t               } }
    rule("\n",      :comment)         { pop_state                         }
  end
end