Skip to content

Instantly share code, notes, and snippets.

@nicolaracco
Created April 15, 2015 09:41
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save nicolaracco/d60efa116c65262b8973 to your computer and use it in GitHub Desktop.
Save nicolaracco/d60efa116c65262b8973 to your computer and use it in GitHub Desktop.
example of a tokenizer and a parser (very basic, even operator precedence is absent)
class Parser
def parse expression
tokenizer = Tokenizer.new expression
first_value = read_next_number tokenizer
while tokenizer.look_next_token
operator = read_next_operator tokenizer
second_value = read_next_number tokenizer
first_value = operate(first_value, operator, second_value)
end
first_value
end
def read_next_number tokenizer
token = tokenizer.next_token
raise "Unexpected #{token.type} - expected number" if token.type != :number
token.value
end
def read_next_operator tokenizer
token = tokenizer.next_token
raise "Unexpected #{token.type} - expected operator" if token.type != :operator
token.value
end
def operate first_value, operator, second_value
first_value.send operator, second_value
end
end
class Token
attr_reader :type, :value
def initialize type, value
@type, @value = type, value
end
end
class Tokenizer
OPERATORS = ['+', '-', '*', '/']
NUMBER_REGEXP = /^\d*(\.\d+)?$/
attr_reader :tokens, :cursor
def initialize expression
tokenize expression
@cursor = -1
end
def current_token
return nil if @cursor == -1
@tokens[cursor]
end
def next_token
@cursor += 1
current_token
end
def look_next_token
@tokens[cursor + 1]
end
private
def tokenize expression
@tokens = []
expression_to_parse = expression.strip
while expression_to_parse.size > 0
@tokens << read_next_token(expression_to_parse)
end
end
def read_next_token expression_to_parse
expression_to_parse.strip!
next_char = expression_to_parse.slice 0
if OPERATORS.include? next_char
read_next_operator_token expression_to_parse
elsif next_char =~ NUMBER_REGEXP
read_next_numeric_token expression_to_parse
else
raise "Unknown token starting with '#{next_char}'"
end
end
def read_next_operator_token expression_to_parse
Token.new :operator, expression_to_parse.slice!(0)
end
def read_next_numeric_token expression_to_parse
numeric_value = expression_to_parse.slice! 0
next_char = expression_to_parse.slice 0
while next_char && "#{numeric_value}#{next_char}" =~ NUMBER_REGEXP
numeric_value << expression_to_parse.slice!(0)
end
Token.new :number, numeric_value.to_f
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment