Skip to content

Instantly share code, notes, and snippets.

@blainegarrett
Last active May 9, 2017 04:00
Show Gist options
  • Save blainegarrett/deabc27a3486d188395960516749ad31 to your computer and use it in GitHub Desktop.
Save blainegarrett/deabc27a3486d188395960516749ad31 to your computer and use it in GitHub Desktop.
Divining Rod Survey Data Sync
import csv
import requests # installed (pip install requests)
import json
INPUT_FILE = './data/diving-rods-survey-data-2017-04-24.csv'
BATCH_SIZE = 100
PREF_API_URL = 'http://localhost:9090/api/rest/v1.0/preferences'
def submit_pref_data(data):
headers = {u'Content-Type': u'application/json'}
r = requests.post(PREF_API_URL, data=json.dumps(data), headers=headers)
if (not r.status_code == 200):
raise Exception(r.text)
def batch(iterable, n=1):
l = len(iterable)
for ndx in range(0, l, n):
yield iterable[ndx:min(ndx + n, l)]
def generate_transaction_data():
preference_data = []
with open(INPUT_FILE, 'rb') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in reader:
user_col_chunks = row[0].split(':') # ['survey', 'user', '9', 'likes']
is_like = user_col_chunks[3] == 'likes'
user_id = 'survey_user' + user_col_chunks[2]
for i in xrange(1, len(row), 1):
preference_data.append({
"item_id": row[i],
"pref": is_like,
"timestamp": None,
"user_id": user_id
})
print 'Collected %s prefs' % len(preference_data)
print 'Submitting preferences %s at a time.' % BATCH_SIZE
cur_batch = 0
for prefs_data in batch(preference_data, BATCH_SIZE):
print "* Sending Batch #%s " % cur_batch
submit_pref_data(prefs_data)
cur_batch += 1
print "Done! "
if __name__ == "__main__":
generate_transaction_data()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment