spark-logs-cleaner.py

import re
import glob

for in_file in glob.glob('/home/owen/logs/some-spark-job-log*.log'):
    out_file = f'{in_file}.p'

    with open(in_file, mode='r') as f:
        with open(out_file, mode='w') as out:
            for line in f:                
                if len(line) > 2:
                    with_stage_removed = re.sub('\[Stage.*?\]','',line, flags=re.DOTALL)
                    if len(with_stage_removed.strip()) > 0:
                        out.write(with_stage_removed)