test_scanner.py

import collections
import difflib
import pathlib
import dill as pickle
import scanner
import sys
import utils

Token = collections.namedtuple('Token', 'line col type string virtual')

def reify_token(token):
    return Token(token.line, token.col, token.type, token.string, token.virtual)

def virtual_repr(token):
    if token.virtual:
        return ':' + ('*' * token.virtual)
    return ''

def extra_repr(token):
    if token.type in ['dedent', 'indent']:
        return f':{len(token.string)}'
    if token.type in utils.variable_content_tokens:
        return f':{token.string!r}'
    return ''

def pretty_token(token):
    base = f'{token.line}:{token.col}:{token.type}'
    return f'({base}{extra_repr(token)}{virtual_repr(token)})'


def print_diff(opcodes, expected, actual):
    for tag, i1, i2, j1, j2 in opcodes:
        if tag == 'equal':
            continue
        print(tag, i1, i2, j1, j2)
        if tag == 'delete':
            for t in actual[i1:i2]:
                print('-', pretty_token(t))
        if tag == 'insert':
            for t in expected[j1:j2]:
                print('+', pretty_token(t))
        if tag == 'replace':
            for t in actual[i1:i2]:
                print('-', pretty_token(t))
            for t in expected[j1:j2]:
                print('+', pretty_token(t))

cwd = pathlib.Path.cwd()

keywords, tokens = open('keywords.json'), open('tokens.json')
regexp = scanner.Regexp.from_files(keywords, tokens)
for path in (cwd / 'examples').glob('*.b'):
    filename = path.relative_to(cwd / 'examples')
    expected_filename = (cwd / 'tests' / filename).with_suffix('.out')
    content = path.open().read()
    tokens = [reify_token(t) for t in scanner.scan(0, regexp, content)]
    if sys.argv[1] == 'load':
        with open(expected_filename, 'rb') as f:
            expected = pickle.load(f)
        sm = difflib.SequenceMatcher(a=expected, b=tokens)
        opcodes = sm.get_opcodes()
        if opcodes != [('equal', 0, len(tokens), 0, len(tokens))]:
            print(f'different result for {filename}')
            print_diff(opcodes, tokens, expected)
    if sys.argv[1] == 'save':
        with open(expected_filename, 'wb') as f:
            pickle.dump(tokens, f)
        print(f'saved {len(tokens)} tokens for {filename}')