problem with python scope.

lex.py

"""
there is no way good way to make this append() function, which is unfortuante because we want the two lines in it to be atomic.
"""

def _lex(line):
    tokens = []
    token = ""
    whitespace = {' '}
    escape_chars = {'\\'}
    escape = False
    separators = ";"
    quotes = {"'", '"'}
    quote = None

    def append():
        # global token ?
        tokens.append(token)
        token = ""

    for char in line:
        if quote is None:
            if escape:
                token += char
                escape = False
            elif char in whitespace:
                append()
            elif char in quotes:
                quote = char
            elif char in escape_chars:
                escape = True
            elif char in separators:
                append()
                token = char
                append()
            else:
                token += char
        elif char == quote:
            append()
            quote = None
        else:
            token += char
    if quote is not None:
        raise ValueError("No closing quotation")
    append()
    return tokens


def lex(line):
    """

    >>> lex("ls")
    ['ls']
    >>> lex("cd /; pwd")
    ['cd', '/', ';', 'pwd']
    >>> lex("'cd /; pwd'")
    ['cd /; pwd']

    """
    return list(_lex(line))