Python Implementation of LL(1) Lexer based loosely on Language Implementation Patterns

utils.py

class Peekable:
    def __init__(self, input_, k, sentinel=None):
        self.sentinel = sentinel
        self._k = k
        self._stream = iter(input_)
        self._peek = [next(self._stream, sentinel) for _ in range(k)]

    def __getitem__(self, n):
        if isinstance(n, int) and n >= self._k:
            raise IndexError(f"Invalid lookahead index {n} on Peekable with k={self._k}")
        return self._peek[n]

    def __iter__(self):
        return self

    def __next__(self):
        if self._peek[0] == self.sentinel:
            raise StopIteration
        res = self._peek[0]
        self._peek = self._peek[1:]
        self._peek.append(next(self._stream, self.sentinel))
        return res

xlexer.py

import string

from utils import Peekable
import xtokens as t

def lex(char_iterable):
    stream = Peekable(char_iterable, 1)
    return _lex(stream)

def _lex(stream):
    while True:
        match stream[0]:
            case stream.sentinel:
                yield t.Eof()
                break
            case '[':
                next(stream)
                yield t.Lbrack()
            case ']':
                next(stream)
                yield t.Rbrack()
            case '=':
                next(stream)
                yield t.Equal()
            case ',':
                next(stream)
                yield t.Comma()
            case c if _is_letter(c):
                yield _lex_ident(stream)
            case c if c in string.whitespace:
                next(stream)
            case c:
                raise ValueError(f"Invalid character {c}")

def _lex_ident(stream):
    cs = []
    while _is_letter(stream[0]):
        cs.append(next(stream))
    return t.Ident(''.join(cs))

def _is_letter(c):
    return c in string.ascii_letters

xtokens.py

from dataclasses import dataclass

@dataclass
class Lbrack:
    pass

@dataclass
class Rbrack:
    pass

@dataclass
class Equal:
    pass

@dataclass
class Comma:
    pass

@dataclass
class Eof:
    pass

@dataclass
class Ident:
    s : str