split reviews into multiple parts based on max_token size

split_reviews_into_multi_parts.py

# split reviews into multiple parts based on max_token size, appending one sentence at a time until the part hits the max token limit

def split_rev(rev, max_tokens = 384):


    rev_sentences = rev.split('.')


    parts_list =  []

    curr_part_tokens = []

    for sentence in rev_sentences:

        if len(curr_part_tokens) == 0:
            curr_part_tokens.extend(sentence.split()[:max_tokens])
            continue

        if len(curr_part_tokens) + len(sentence.split()) + 1 <= max_tokens:
            curr_part_tokens.extend(['.'] + sentence.split()[:max_tokens])

        else:
            parts_list.append(" ".join(curr_part_tokens))
            curr_part_tokens = sentence.split()[:max_tokens]


    if len(curr_part_tokens) > 0:
        parts_list.append(" ".join(curr_part_tokens))

    parts_list = [x.replace(' .', '.') for x in parts_list]

    # print(parts_list)
    # print(len(parts_list))
    # print([len(x.split()) for x in parts_list])

    return parts_list