Optimized charmap generation

gen_charmap.py

import sys
import time
import unicodedata


def gen_charmap_new():
    current_cat = None
    current_cat_start = None
    ranges = []

    for i in range(0, sys.maxunicode + 1):
        cat = unicodedata.category(chr(i))
        if cat != current_cat:
            if current_cat is not None:
                ranges.append((current_cat, current_cat_start, i - 1))
            current_cat = cat
            current_cat_start = i
    ranges.append((current_cat, current_cat_start, sys.maxunicode))

    tmp_charmap = {}
    for cat, start, end in ranges:
        rs = tmp_charmap.setdefault(cat, [])
        rs.append([start, end])

    return tmp_charmap


def gen_charmap_old():
    tmp_charmap = {}
    for i in range(0, sys.maxunicode + 1):
        cat = unicodedata.category(chr(i))
        rs = tmp_charmap.setdefault(cat, [])
        if rs and rs[-1][-1] == i - 1:
            rs[-1][-1] += 1
        else:
            rs.append([i, i])
    return tmp_charmap

t1 = time.time()
cm1 = gen_charmap_old()
t2 = time.time()
cm2 = gen_charmap_new()
t3 = time.time()

print('Charmaps equal', cm1 == cm2)
print('Generating charmap (old) took', t2 - t1)
print('Generating charmap (new method) took', t3 - t2)

output.txt

[I]  ~/o/h/hypothesis-python (vi)> time python3 gencharmap.py
Charmaps equal True
Generating charmap (old) took 0.869232177734375
Generating charmap (new method) took 0.4008798599243164
        1.38 real         1.31 user         0.04 sys