Testing of lib2to3.pgen2.tokenize for Python 3.6

test_tokenize.py

import re

# ----------------------------------------------------------------
# Define regular expressions
# ----------------------------------------------------------------

def group(*choices): return '(' + '|'.join(choices) + ')'
def maybe(*choices): return group(*choices) + '?'

Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
Octnumber = r'0[oO]_?[0-7]+(?:_[0-7]+)*[lL]?'
Decnumber = r'[1-9]_?\d+(?:_\d+)*[lL]?'

Exponent = r'[eE][-+]?\d+(?:_\d+)*'
Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d*(?:_\d+)*') + maybe(Exponent)
Expfloat = r'\d+(?:_\d+)*' + Exponent
Floatnumber = group(Pointfloat, Expfloat)

Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')

# ----------------------------------------------------------------
# Test integer types
# ----------------------------------------------------------------

def check(regex, valid, invalid):
    for case in valid:
        assert re.match(regex + '$', case), '{} should be valid!'.format(repr(case))

        for case in invalid:
            assert re.match(regex + '$', case) is None, '{} should not be valid!'.format(repr(case))

    return True


valid = [
    '0x10',
    '0x1_0',
    '0x1_00',
    '0x1_0_0',
    '0x1_00_1',
    '0x1_00_1000_1',
    '0x_1_0',
    '0x_1_000',
    '0x_1_000_1',
    '0x_10',
    '0x_10_00',
]

invalid = [
    '0x10_',
    '0x__10_',
    '0x1__0_',
    '0x100__00_',
    '0x__10',
    '0x1__0',
    '0x100__00',
]

check(Hexnumber, valid, invalid)
check(Binnumber, ['0b' + w[2:] for w in valid], ['0b' + w[2:] for w in invalid])
check(Octnumber, ['0o' + w[2:] for w in valid], ['0o' + w[2:] for w in invalid])
check(Decnumber, [w for w in [w[2:] for w in valid] if not w.startswith('_')],
      [w[2:] for w in invalid])
check(Imagnumber, [w for w in [w[2:] + 'j' for w in valid] if not w.startswith('_')],
      [w[2:] + 'j' for w in invalid])


# ----------------------------------------------------------------
# Test floating point types
# ----------------------------------------------------------------

valid = [
    '30.0',
    '030.0',
    '30.',
    '030.',
    '3_0.0',
    '03_0.0',
    '3_0.',
    '3.0_1',
    '.0_1',
    # Exponents
    '1e30',
    '1e3_0',
    '1e-3_0',
    '1e+3_0',
    '1_0e+3_0',
    '1_0_1e-30_0',
]

invalid = [
    '30',
    '_30.',
    '_30.0',
    '30._0',
    '3._',
    '3__0.',
    '3__0.0',
    '3__0',
    # Exponents
    '1e30.0',
    '1e_30',
    '1e30_'
]

check(Floatnumber, valid, invalid)
check(Imagnumber, [w + 'j' for w in valid], [w + 'j' for w in invalid[1:]])