Work frequency in a python codebase.

zipf.py

import tokenize, collections, pathlib, pprint

counter = collections.Counter()
source_path = pathlib.Path('.')

for filename in source_path.glob('**/*.py'):
    print(f'Scanning tokens {filename} ...')
    tokens = tokenize.tokenize(open(filename, 'rb').__next__)
    for token in tokens:
        if token.type == 1 or token.type == '3' and ' ' not in token.string:
            counter[token.string] += 1

pprint.pprint(counter)
print(f'len: {len(counter)}')