Tiny regex based tokenizer in python

__main__.py

from .lexer import Lexer

code = '''
(define (fib x)
    (if (lt x 2)
        x
        (add (fib (sub x 1))
             (fib (sub x 2)))))
'''

lexer = Lexer(  # **kwargs do preserve order
    WHITESPACE = r'\s+',
    PAREN      = r'[\(\)]',
    DEFINE     = r'define',
    IF         = r'if',
    NAME       = r'[a-zA-Z]\w*',
    NUMBER     = r'\d+',
)

list(lexer.tokenize(code))[:10]
# [('WHITESPACE', '\n'),
#  ('LPAREN', '('),
#  ('DEFINE', 'define'),
#  ('WHITESPACE', ' '),
#  ('LPAREN', '('),
#  ('NAME', 'fib'),
#  ('WHITESPACE', ' '),
#  ('NAME', 'x'),
#  ('RPAREN', ')'),
#  ('WHITESPACE', '\n    ')]

lexer.py

from re import compile

class Lexer:
    def __init__(self, **rules):
        self.rules = [(k, compile('^' + v)) for k, v in rules.items()]

    def tokenize(self, code):
        while code:
            match, code = self.tokenize_one(code)
            yield match

    def tokenize_one(self, code):
        for name, rule in self.rules:
            m = rule.search(code)
            if m: return (name, code[:m.end()]), code[m.end():]
        raise Exception("Can't match:\n" + code)