Normalize spaces before importing as dataframes

normalize_spaces.py

#%%
import os
import re

os.chdir('data_directory')
raw_files = [x for x in os.listdir()]

for raw in raw_files:
    inputfile = raw
    outputfile = raw.split('.')[0]+'_clean.txt'
    
    with open(inputfile, 'r') as f:
        lines = f.readlines()
    lines = [x.strip('\n') for x in lines]

    data_rows = []
    for line in lines:
        cleanline = re.sub('\s+', ' ', line).strip()
            data_rows.append(cleanline)
    
    with open(outputfile, 'a') as f:
        for obs in data_rows:
            f.write(obs + '\n')
        f.close()