Chunk a large text by a regex & nr of times the regex matches. Could be useful for text processing where you want to use the chunks for same regex as the tokenizer

chunk_text_by_regex.py

import re

def chunk_text_by_regex(text, pattern, nr_matches):
    prev = 0
    chunks = []
    for i, match in enumerate(re.finditer(pattern, text), start=1):
        if i % nr_matches == 0:  
            chunks.append(text[prev:match.start()].strip())
            prev = match.start()
    if prev < len(text):
        chunks.append(text[prev:].strip())
    return chunks

text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis tristique nisi eu turpis vulputate, non venenatis elit consectetur. Curabitur consectetur vehicula mattis. Etiam laoreet libero sit amet neque blandit efficitur. Curabitur faucibus eleifend diam, eu placerat nisl laoreet in. Nunc dolor quam, aliquet id quam vitae, tincidunt feugiat neque. Nullam a magna feugiat, iaculis lectus in, iaculis orci. Quisque risus enim, ultricies sed mauris vel, eleifend aliquet ex. Proin sit amet fermentum urna. Mauris quis ex justo. Curabitur fermentum urna et sapien porttitor accumsan."
regex = r"\b[a-zA-Z0-9-']+\b"
chunks = chunk_text_by_regex(text, regex, 5)

print(chunks)