Skip to content

Instantly share code, notes, and snippets.

@aki237
Created November 22, 2017 09:27
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save aki237/2ebcb726d5b67b7039d3f4a599b5b9b8 to your computer and use it in GitHub Desktop.
Save aki237/2ebcb726d5b67b7039d3f4a599b5b9b8 to your computer and use it in GitHub Desktop.
Tokeninzing lisp in python
#!/usr/bin/python3
class Tokenizer:
OPEN_BRACKET = '('
CLOSE_BRACKET = ')'
"""Tokenizer class is used to tokenize a lisp statement into separate symbols"""
def __init__(self, expression):
"""Class constructor used to initialize some member values"""
self.exp = expression.strip()
self.offset = 0
if self.exp[self.offset] is not self.OPEN_BRACKET:
raise Exception("Unexpected character '" + self.exp[self.offset] + "' found at 0th index. Expected '" + self.OPEN_BRACKET + "'")
def match_pair(self, starter, ender):
"""Returns the index of the matching pair for a given 2 characters"""
moff = self.offset
if self.exp[self.offset] is not starter:
return -1
count = 1
self.offset += 1
while self.offset < len(self.exp):
if self.exp[self.offset] is starter:
count += 1
if self.exp[self.offset] is ender:
count -= 1
if count is 0:
xx = self.offset
self.offset = moff
return xx
self.offset += 1
self.offset = moff
return -1
def get_tokens(self):
"""Returns a list containing the tokens for the lisp statement."""
tokens = []
index = self.match_pair(self.OPEN_BRACKET, self.CLOSE_BRACKET)
if index is -1:
raise Exception("Bracket match not found")
temp = ""
while (self.offset + 1) < len(self.exp):
self.offset += 1
curoff = self.offset
ch = self.exp[curoff]
if ch is self.OPEN_BRACKET:
mindex = self.match_pair(self.OPEN_BRACKET, self.CLOSE_BRACKET)
tokens.append(self.exp[curoff:mindex+1])
self.offset = mindex
continue
if ch is ' ':
if temp.strip() is not "":
tokens.append(temp)
temp = ""
continue
if ch.strip() is '':
if temp.strip() is not "":
tokens.append(temp)
temp = ""
continue
temp += ch
return tokens
print(Tokenizer(input().strip()).get_tokens())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment