lex.py

def lex(text):
    sym = r'[\w+\-*/=<>]+'
    toks = (
        r';.*?\n\s*',
        lambda m: None,
        r'-?(\d+)?\.\d+(e-?\d+)?\s*',
        lambda m: ['val', float(m.group(0))],
        r'-?\d+\s*',
        lambda m: ['val', int(m.group(0))],
        r"'((?:[^'\\]|\\.)*)'\s*",
        lambda m: ['val', m.group(1)],
        r'([@#])(\d+)\s*',
        lambda m: [{'@': 'arr', '#': 'map'}[m.group(1)], int(m.group(2))],
        r'~(~?)(\d*)\s*',
        lambda m: ['tcall' if m.group(1) else 'call'] + ([int(m.group(2))] if m.group(2) else []),
        r'\.(%s)?\s*' % sym,
        lambda m: ['label', m.group(1)] if m.group(1) else ['ret'],
        r'([%%$=!?])(%s)\s*' % sym,
        lambda m: [{'=': 'set', '$': 'get', '%': 'fn', '!': 'jmp', '?': 'jmpf'}[m.group(1)], m.group(2)],
        r':(%s(:?,%s)*)\s*' % (sym, sym),
        lambda m: ['args'] + m.group(1).split(',')
    )
    compiled = [(re.compile(regex), action) for regex, action in pairs(toks)]
    pos = 0
    text = text.strip()
    while pos < len(text):
        for regex, action in compiled:
            m = regex.match(text, pos)
            if m:
                pos += len(m.group(0))
                tok = action(m)
                if tok: yield tok; break
        else:
            print('illegal character %s' % repr(text[pos]))
            pos += 1