Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
from sre_parse import Pattern, SubPattern, parse as sre_parse
from sre_compile import compile as sre_compile
from sre_constants import BRANCH, SUBPATTERN
class Scanner(object):
def __init__(self, tokens, flags=0):
subpatterns = []
pat = Pattern()
pat.flags = flags
self._tokens = []
for token_name, regex in tokens:
subpatterns.append(SubPattern(pat, [
(SUBPATTERN, (len(subpatterns) + 1, sre_parse(regex, flags))),
]))
self._tokens.append(token_name)
pat.groups = len(subpatterns) + 1
self._scanner = sre_compile(SubPattern(pat, [
(BRANCH, (None, subpatterns))
])).scanner
def scan(self, string, skip=False):
scanner = self._scanner(string)
match = skip and scanner.search or scanner.match
i = 0
while 1:
m = match()
if not m:
break
j = m.end()
if i == j:
break
token = self._tokens[m.lastindex - 1]
yield token, m
i = j
if i < len(string):
raise EOFError(i)
def test_precise():
scanner = Scanner([
('whitespace', r'\s+'),
('plus', r'\+'),
('minus', r'\-'),
('mult', r'\*'),
('div', r'/'),
('num', r'\d+'),
('paren_open', r'\('),
('paren_close', r'\)'),
])
for token, match in scanner.scan('(1 + 2) * 3'):
print (token, match.group())
def test_lenient():
scanner = Scanner([
('bold', '\*\*'),
('link', '\[\[(.*?)\]\]'),
])
input_text = 'Hello **World**! [[Stuff]]'
pos = 0
for token, match in scanner.scan(input_text, skip=True):
skipped = input_text[pos:match.start()]
if skipped:
print (None, skipped)
print (token, match.group())
pos = match.end()
skipped = input_text[pos:]
if skipped:
print (None, skipped)
if __name__ == '__main__':
print 'Precise:'
test_precise()
print 'Lenient:'
test_lenient()
Precise:
('paren_open', '(')
('num', '1')
('whitespace', ' ')
('plus', '+')
('whitespace', ' ')
('num', '2')
('paren_close', ')')
('whitespace', ' ')
('mult', '*')
('whitespace', ' ')
('num', '3')
Lenient:
(None, 'Hello ')
('bold', '**')
(None, 'World')
('bold', '**')
(None, '! ')
('link', '[[Stuff]]')
@singingwolfboy

This comment has been minimized.

Copy link

singingwolfboy commented Nov 18, 2015

what is this magic

@mitsuhiko

This comment has been minimized.

Copy link
Owner Author

mitsuhiko commented Nov 18, 2015

This gem was hidden in a similar form in re.Scanner in the stdlib for the last 15 years. Why was I not informed! This changes everything about lexing in Python.

@foresmac

This comment has been minimized.

Copy link

foresmac commented Nov 18, 2015

So, you're gonna give a talk about this at PyCon so I know what the heck this means, right?

@tylerdave

This comment has been minimized.

Copy link

tylerdave commented Nov 18, 2015

Just searched and found this post form 2005: http://code.activestate.com/recipes/457664-hidden-scanner-functionality-in-re-module/
Don't see it in the docs in 2015 😢

@mitsuhiko

This comment has been minimized.

Copy link
Owner Author

mitsuhiko commented Nov 18, 2015

@tylerdave: re.Scanner unfortunately is not all that useful by itself because it creates a list and does not support scanning ahead over unlexed contents which this one can do.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.