eventlogging_count_field.py

# export PYTHONPATH=/srv/deployment/eventlogging/eventlogging
## https://github.com/jsocol/pystatsd/issues/24
# export DJANGO_SETTINGS_MODULE="DO NOT USE" 
#
# python count_field.py <topic> <field>
#
from eventlogging import get_reader

import argparse
from pprint import pprint

ap = argparse.ArgumentParser()
ap.add_argument('topic')
ap.add_argument('field')
args = ap.parse_args()

kafka_uri = 'kafka:///kafka1012.eqiad.wmnet:9092?zookeeper_connect=conf1001.eqiad.wmnet:2181/kafka/eqiad&topic=%s' % args.topic

counts = {}

for e in get_reader(kafka_uri):
    if args.field in e['event']:
        value = e['event'][args.field]
        count = counts.setdefault(value, 0)
        counts[value] += 1
    pprint(counts)

export PYTHONPATH=/srv/deployment/eventlogging/eventlogging
export DJANGO_SETTINGS_MODULE="DO NOT USE" 
python ./ereader.py   eventlogging_GeoFeatures feature

...
{u'GeoHack': 57, u'WIWOSM': 2, u'WikiMiniAtlas': 3}
{u'GeoHack': 58, u'WIWOSM': 2, u'WikiMiniAtlas': 3}
{u'GeoHack': 59, u'WIWOSM': 2, u'WikiMiniAtlas': 3}
{u'GeoHack': 60, u'WIWOSM': 2, u'WikiMiniAtlas': 3}
{u'GeoHack': 61, u'WIWOSM': 2, u'WikiMiniAtlas': 3}
{u'GeoHack': 62, u'WIWOSM': 2, u'WikiMiniAtlas': 3}