datum pseudo

gistfile1.txt

import datum
from ais import Geocoder


# Connect to dataset. This could be SFTP, local CSV, GeoDB2, whatever.
props = datum.table('sftp://user:password@secure-ftp.host.com/path/to/data.csv')

# Rename fields.
props.map_fields({
    'acct':     'account_num',
    'addr':     'address',
    'owner1':   'owner_1',
    'owner2':   'owner_2',
    'mv':       'market_value',
    'mv_date':  'market_value_date',
})

# Exclude props without a market value. The `warn` flag writes failing rows to stderr.
props.exclude({'market_value': None}, warn=True)

# Round down market values by passing in the `int` type.
# (Technically these are already ints in OPA, but just by way  example...)
props.transform('market_value', int)

# Remove trailing whitespace from addresses. Pass in the name of the built-in string function.
props.transform('address', 'rstrip')

# Strip last two chars from market value date.
props.transform('market_value_date', lambda x: x[:-2])

# Make owner 1 and 2 upper case.
props.transform(['owner_1', 'owner_2'], 'upper')

# Add a field concatenating owner 1 and 2.
props.add_field('owners', lambda row: row['owner_1'] + '|' + row['owner_2'])

# Geocode. We start by wrapping the AIS geocoding API.
g = Geocoder('http://api.phila.gov/ais/geocode/')

# Now pass in the name of the address field and a function that will return tuples of XYs.
props.geocode('address', g.geocode)

# Write out.
props.pipe('oracle-stgeom://user:password@host:1521/gis_opa.properties')