.gitignore

*.pyc

csv2dict.py

import csv


def dictify(filename):
	"""
	turn data from CSV file into a list of dictionaries
	"""
	f = open(filename, "rU")

	items = []
	for row in unicode_csv_reader(f):
		try:
			item = dict([(k, row[i]) for i, k in enumerate(keys)])
			item["_source"] = filename # XXX: not a generic requirement
			items.append(item)
		except NameError: # first row
			keys = row

	f.close()
	return items


# Unicode fixes as per documentation
def unicode_csv_reader(data, dialect=csv.excel, **kwargs):
	"""
	expects Unicode input, temporarily encoding it as UTF-8
	"""
	reader = csv.reader(utf8_encoder(data), dialect=dialect, **kwargs)
	for row in reader:
		yield [unicode(cell, "utf-8", "replace") for cell in row]


def utf8_encoder(data):
	for line in data:
		yield line

csv2tiddlers.py

#!/usr/bin/env python

"""
Usage:
  $ ./csv2tiddlers.py <host> <bag> <filename> <titlefields> <textfields> <fieldmap> <tagsmap>

Example:
  $ ./csv2tiddlers.py http://localhost:8080 common sample.csv "colA;colC" "colD;colE" "colB=foo;colF=bar"
"""

import sys

from csv2dict import dictify
from tiddler import put


def main(args):
	"""
	arguments are host, bag, filename, titlefields, textfields and fieldmap

	fields are a list of semicolon-separated column names whose values will be
	concatenated for the respective tiddler attribute

	fieldmap provides a way to translate column names to tiddler fields
	tagsmap provides a way to translate column values into tags
	"""
	args = [unicode(arg, "utf-8", "replace") for arg in args]
	try:
		host = args[1]
		bag = args[2]
		filename = args[3]
		titlefields = args[4].split(";")
		textfields = args[5].split(";")
	except IndexError, exc:
		print "ERROR: missing argument", exc
		return False
	try:
		fieldmap = dict([i.split("=") for i in args[6].split(";")])
	except IndexError, exc:
		fieldmap = {}
	try:
		tagsmap = args[7].split(";")
	except IndexError, exc:
		tagsmap = []

	items = dictify(filename)
	put_tiddlers(items, host, bag, titlefields, textfields, fieldmap, tagsmap)

	return True


def put_tiddlers(items, host, bag, titlefields, textfields, fieldmap, tagsmap):
	for item in items:
		values = []
		for field in titlefields:
			values.append(item[field])
		title = "_".join(values)

		try:
			values = []
			for field in textfields: # TODO: DRY
				values.append(item[field])
			text = "\n\n".join(values)
		except KeyError:
			text = None # XXX: invalid?

		for column, field in fieldmap.items(): # XXX: inefficient!?
			try: # XXX: temporary workaround
				item[field] = item[column]
				del item[column]
			except KeyError:
				pass

		try:
			tagsCol, tagsDelim = tagsmap
			tags = item[tagsCol].split(tagsDelim)
		except (ValueError, KeyError), exc:
			tags = []

		tiddler = {
			"text": text,
			"tags": tags,
			"fields": item
		}

		message = "sending tiddler: %s ..." % title.encode("utf-8")
		sys.stdout.write(message)
		status = put(host, bag, title, tiddler)
		print "\t[%s]" % status


if __name__ == "__main__":
	status = not main(sys.argv)
	sys.exit(status)

tiddler.py

"""
PUT a tiddler.
"""

import urllib2
import httplib2

try:
	import json
except ImportError, exc:
	import simplejson as json


def put(host, bag, title, tiddler):
	"""
	send a dictionary representing a tiddler to a TiddlyWeb server
	"""
	http = httplib2.Http()
	bag = urllib2.quote(bag.encode("utf-8"))
	title = urllib2.quote(title.encode("utf-8"))
	uri = "%s/bags/%s/tiddlers/%s" % (host, bag, title)
	response, content = http.request(
		uri,
		method="PUT",
		headers={ "Content-Type": "application/json" },
		body=json.dumps(tiddler)
	)
	return response["status"]

zip2geo.py

#!/usr/bin/env python

import sys
import urllib2
import httplib2


URI = "http://www.freethepostcode.org/geocode?postcode=%s"

DATA = """
WC1N 1HT
SW7 4DN
SM1 2RF
E16 1EA
UB7 9NA
UB7 0DP
SW1E 6AF
E4 8ST
SE3 0RW
KT9 2NE
SW19 2BH
NW2 1LP
E16 1AL
W14 9NS
HA9 8HQ
SE1 9SG
W1W 5EE
WC1X 9HX
UB3 5AJ
W6 OQU
TW8 8AQ
E15 2NE
W1X 6NE
DA5 1ND
E16 2EE
EC4V 6DB
UB7 0JU
E1W 3HN
EC1V 9LN
SE10 8NA
SW18 1EG
SW19 5BA
UB4 0HF
E16 1SL
KT9 2NE
N12 0QZ
HA8 5AQ
E1 6BF
SE1 0JQ
NW1 2QE
IG11 7BA
SW1P 4DD
NW1 7BY
SE1 7PB
SW8 1RN
IG11 7BA
CR0 5RB
N1 0UY
TW6 3AF
W2 4RT
W6 9LU
SE1 0JA
UB7 9HB
RM13 9ED
NW7 3HU
TW2 6LS
SE16 5PR
WC2B 5RE
IG11 7BA
CR0 4RQ
UB3 1RW
SW19 2RD
SE1 9EF
W8 5SR
EN3 7NJ
E16 1AB
E1 7DB
NW2 3ED
NW1 2AJ
UB6 8TR
SE1 7BJ
CR0 9XY
SW6 3JD
SE1 9HH
IG1 1BA
KT6 7EL
W1J 7QY
E16 3BY
WC2B 4DD
UB3 5AL
SW1V 1EQ
SW6 1UD
TW13 4EX
E14 2AA
W1S 4BP
WC1H 0DG
WC1H 9EU
WC1N 1DB
WC1B 5BB
SM4 5PH
E14 9SH
EC1Y 1AE
RM1 1XJ
SW11 3SA
W1H 7EH
WC1H 9PJ
SW6 1HS
W2 1JU
E15 4LJ
N1 9LA
TW8 0BB
E16 2BZ
WC1X 8BH
TW1 4RB
RM1 3EN
SE1 7LS
W6 8DR
SM5 3PE
RM6 6QU
NW3 4RB
SE1 3LP
EC1V 8DS
NW1 2LW
E16 1AA
SE10 8JA
SW7 4QH
SW7 4LH
SW1X 9NU
W1K 2HP
W1C 1BY
W1T 5AY
WC2H 7LH
WC1B 3BL
WC2H 7HL
WC1B 3QD
W1J 8LT
WC2H 9HD
W1H 6PA
SW7 5BT
E14 8RS
EC3N 2BQ
WC1B 4AR
W1W 5NB
W1J 7BN
W2 4RJ
W2 1EE
SE1 2BY
W1K 1BE
SW1A 2TS
EC2M 7QN
W1H 7BH
W1H 7BY
W1K 1LB
W1B 1JA
SW1A 1NJ
SW6 2GA
W1J 7BJ
E16 1RH
W8 4PT
IG2 7RH
SE10 0GD
W3 6UP
E1 6JQ
W1G 8DN
WC1B 3NN
W1K 6JP
NW6 5RP
W1H 5DN
NW3 3ST
E14 4ED
W1K 7AA
N1 9AA
W1J 5EB
SW1W 0PS
W8 5DL
WC1B 4BH
W14 0AX
NW1 6JQ
SW1X 7RN
W1H 7DL
WC1A 2SD
WC2N 5HX
NW1 2LP
W2 3NR
W2 3HL
W1D 6BZ
SW1A 2EJ
E1W 1LD
WC2H 7HG
SW1W 0SJ
SW1W 0QT
WC1X 9QN
IG4 5DR
NW1 6SE
UB3 5AW
SE10 9JN
W1U 6LJ
SE16 5HW
E1 8BB
IG11 7BA
E14 9PE
SW1E 6DU
SW1V 2BJ
"""

def main(args):
	args = [unicode(arg, "utf-8") for arg in args]

	for postcode in DATA.strip().split("\n"):
		print resolve(postcode)


def resolve(postcode):
	uri = URI % urllib2.quote(postcode)
	http = httplib2.Http()
	response, content = http.request(uri, method="GET")

	try:
		loc = content.split("\n")[1]
		loc = loc.split(" ")
		lat, lon = loc[0:2]
	except ValueError, exc:
		print "ERROR:", response, content
		return False
	return "%s: %s, %s" % (postcode, loc[0], loc[1])


if __name__ == "__main__":
	status = not main(sys.argv)
	sys.exit(status)