Script to process vocabulary

get_vocab.py

import pandas as pd

from collections import Counter
from tqdm.contrib.concurrent import process_map

def get_vocab(texts_list, ids_list=None):
    
    def sum_counters(counter_list):

        '''
        Recursive counter with a O(log(n)) Complexity
        Sourced from https://stackoverflow.com/a/62393323
        '''

        if len(counter_list) > 10:

            counter_0 = sum_counters(counter_list[:int(len(counter_list)/2)])
            counter_1 = sum_counters(counter_list[int(len(counter_list)/2):])

            return sum([counter_0, counter_1], Counter())

        else:

            return sum(counter_list, Counter())
    
    ids_list = range(len(texts_list)) if ids_list is None else ids_list
    
    char_counts = process_map(Counter, texts_list, chunksize=1000)
    
    # Document-character counts matrix
    text_char_df = pd.concat([
        pd.DataFrame({ 'id' : ids_list, 'text' : texts_list }),
        pd.DataFrame(char_counts).fillna(0).astype(int)
    ], axis=1)
    
    # Aggregates
    char_aggs = sum_counters(char_counts)
    
    return char_aggs, text_char_df