Parsing Text Files

example.py

import glob
import re
import csv

allfiles = glob.glob('1972_1998_batch/*.sim')

fw = open('psb.csv','w')
dw = csv.DictWriter(fw,["filename","month","electricity","gas"])
dw.writeheader()
fwt = open('psb_total.csv','w')
dwt = csv.DictWriter(fwt,["filename","month","electricity","gas"])
for fname in allfiles:
    f = open(fname)
    d = dict()
    numMatches = 0
    for line in f:
        res = re.search('\(UNITS/MO\) \s* (\d+\.\d) (\s* (\d+\.\d))?',line)
        if res is not None:            
            numMatches = numMatches + 1
            d["month"] = numMatches
            d["electricity"] = res.group(1)
            d["gas"] = res.group(3)
            d["filename"] = fname
            dw.writerow(d)
        res = re.search('\(UNITS/YR\) \s* (\d+\.\d) (\s* (\d+\.\d))?',line)
        if res is not None:     
            numMatches = numMatches + 1
            d["month"] = "total"
            d["electricity"] = res.group(1)
            d["gas"] = res.group(3)
            d["filename"] = fname
            dw.writerow(d)
            dwt.writerow(d)
    f.close()
fw.close()
fwt.close()

# Now, do the PS-E tables

fw = open('pse.csv','w')
dw = csv.DictWriter(fw,["filename","month","misc-equipment"])
dw.writeheader()
for fname in allfiles:    
    f = open(fname)
    d = dict()
    inPSE = False    
    for line in f:
        res = re.search('REPORT- PS-E',line)
        if res is not None:
            inPSE = True
        res = re.search('REPORT- PS-F',line)
        if res is not None:
            inPSE = False
        res = re.search('MISC EQUIPMT',line)
        if (res is not None) and (inPSE):
            all_me = re.findall('\d+\.',line)
            m = 0
            for val in all_me:
                m = m+1
                if m<=12:
                    d["month"] = m
                else:
                    d["month"] = "total"
                d["misc-equipment"] = val
                d["filename"] = fname
                dw.writerow(d)
    f.close()
fw.close()
            
# go over every file
# read in the file
# mush things around
# save it as a csv