Divining Rod Survey Data Sync

divirod_survey_sync.py

import csv
import requests  # installed (pip install requests)
import json

INPUT_FILE = './data/diving-rods-survey-data-2017-04-24.csv'
BATCH_SIZE = 100
PREF_API_URL = 'http://localhost:9090/api/rest/v1.0/preferences'


def submit_pref_data(data):
    headers = {u'Content-Type': u'application/json'}
    r = requests.post(PREF_API_URL, data=json.dumps(data), headers=headers)
    if (not r.status_code == 200):
        raise Exception(r.text)


def batch(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx + n, l)]


def generate_transaction_data():
    preference_data = []

    with open(INPUT_FILE, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in reader:
            user_col_chunks = row[0].split(':')  # ['survey', 'user', '9', 'likes']
            is_like = user_col_chunks[3] == 'likes'
            user_id = 'survey_user' + user_col_chunks[2]

            for i in xrange(1, len(row), 1):
                preference_data.append({
                    "item_id": row[i],
                    "pref": is_like,
                    "timestamp": None,
                    "user_id": user_id
                    })

    print 'Collected %s prefs' % len(preference_data)
    print 'Submitting preferences %s at a time.' % BATCH_SIZE

    cur_batch = 0
    for prefs_data in batch(preference_data, BATCH_SIZE):
        print "* Sending Batch #%s " % cur_batch
        submit_pref_data(prefs_data)
        cur_batch += 1

    print "Done! "


if __name__ == "__main__":
    generate_transaction_data()