Lexer Example

lexer_ex.py

def read_while(self, predicate):
    str = ''
    while not self._data.end() and predicate(self._data.peek()):
        str += self._data.next()

    return str

@staticmethod
def is_string(char, strict=True):
    if strict:
        return char.isalpha()

    return char.isalnum() or char in ['.', '_', ':']

def read_string(self):
    return Token(StringToken, self.read_while(functools.partial(self.is_string, strict=False)))