Process all word documents into a single text file (for GPT training)

ingest.py

#!/usr/bin/env python
import os
import docx2txt

from absl import app
from absl import flags

FLAGS = flags.FLAGS
flags.DEFINE_string("output", "output.txt", "The path of final output.")
flags.DEFINE_string("source", None, "The source path to process.")

flags.mark_flag_as_required("source")

def flush(text):
    text = text.replace("\n\n","\n")
    with open(FLAGS.output, "a") as f:
        f.write(text)

def main(argv):
    del argv
    for root, dirs, files in os.walk(FLAGS.source):
        for file in files:
            if file.endswith(".docx"):
                full_path = os.path.join(root, file)
                text = docx2txt.process(full_path)
                flush(text)
                print(f"Processed {full_path}") 


if __name__ == '__main__':
    app.run(main)