Skip to content

Instantly share code, notes, and snippets.

@Ceasar Ceasar/lex.py
Last active Aug 29, 2015

Embed
What would you like to do?
problem with python scope.
"""
there is no way good way to make this append() function, which is unfortuante because we want the two lines in it to be atomic.
"""
def _lex(line):
tokens = []
token = ""
whitespace = {' '}
escape_chars = {'\\'}
escape = False
separators = ";"
quotes = {"'", '"'}
quote = None
def append():
# global token ?
tokens.append(token)
token = ""
for char in line:
if quote is None:
if escape:
token += char
escape = False
elif char in whitespace:
append()
elif char in quotes:
quote = char
elif char in escape_chars:
escape = True
elif char in separators:
append()
token = char
append()
else:
token += char
elif char == quote:
append()
quote = None
else:
token += char
if quote is not None:
raise ValueError("No closing quotation")
append()
return tokens
def lex(line):
"""
>>> lex("ls")
['ls']
>>> lex("cd /; pwd")
['cd', '/', ';', 'pwd']
>>> lex("'cd /; pwd'")
['cd /; pwd']
"""
return list(_lex(line))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.