chunk_split.py

#!/usr/bin/env python

import os
import sys

input_fn = sys.argv[1]

with open(input_fn, "rt", encoding="UTF-8") as fp:
    input_text = fp.read()

input_text = input_text.replace(" ", "").replace("\n", "")

output_fn_base = os.path.splitext(os.path.basename(input_fn))[0]
output_idx = 1

pos_begin = 0
chunk_len = 4000
margin_len = 200

while True:
    output_fn = f"{output_fn_base} {output_idx:03d}.txt"
    chunk_text = input_text[pos_begin:pos_begin+chunk_len]
    with open(output_fn, "wt", encoding="UTF-8") as fp:
        fp.write(chunk_text)

    output_idx += 1
    pos_begin += chunk_len - margin_len

    if pos_begin > len(input_text):
        break