asos-filter.py

import sys
import re
import fileinput

# This script takes a list of air temperatures extracted from historical METARs
# obtained from https://mesonet.agron.iastate.edu/request/download.phtml.
#
# Use it in the command line with something along the lines of
#     $ python2.7 asos-filter.py asos-1997.csv
# where the parameter is the text file you want to filter through. It only
# supports comma-separated files and will remove the header.
#
# This filters through the CSV and returns only the hourly observations taken at
# :53 past the hour. It will include "measurements" of M indicating data issues.
# It will discard other measurements (such as 5-minute obs and special obs).
#
# Be sure to tweak the regex accordingly. If you're not experienced with regex,
# you can experiment at pythex.org. This script overwrites the file, so it is
# STRONGLY recommended that your regex is rock-solid before running the script!
#
# Adapted from https://stackoverflow.com/a/17221420

filename = sys.argv[1]

regex = '[\w\d]{1,},\d{4}-\d{2}-\d{2} \d{2}:53,-?[\w\d]?\d?.?\d?\d?'

for line in fileinput.input(filename, inplace = True):
	if re.search(regex, line):
		print line,
	else:
		print "",