Skip to content

Instantly share code, notes, and snippets.

@TheFifthFreedom
Last active January 26, 2016 17:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save TheFifthFreedom/e149ab0dcb5bff360ebc to your computer and use it in GitHub Desktop.
Save TheFifthFreedom/e149ab0dcb5bff360ebc to your computer and use it in GitHub Desktop.
# -*- coding: utf-8 -*-
import csv
import json
import time
import requests
class Scraper:
def __init__(self):
self.output_file = csv.writer(open('venmo_output.csv', 'a'))
def pull_data(self):
try:
url = "https://venmo.com/api/v5/public"
return json.loads(requests.get(url).text)
except (requests.exceptions.ConnectionError, ValueError) as e:
return None
def transform_data(self, json_object):
output_array = []
try:
json_array = json_object['data']
for item in json_array:
try:
output_array.append([item['story_id'],item['updated_time'],item['actor']['name'],
item['actor']['picture'],item['actor']['id'],
item['transactions'][0]['target']['name'],
item['transactions'][0]['target']['picture'],
item['transactions'][0]['target']['id'],
item['message'],item['type']])
except TypeError:
pass
except (TypeError, KeyError) as e:
pass
return output_array
def write_data(self, raw_array):
for row in raw_array:
self.output_file.writerow([str(s) for s in row])
if __name__ == '__main__':
scraper = Scraper()
while True:
start_time = int(time.time())
json_object = scraper.pull_data()
raw_array = scraper.transform_data(json_object)
scraper.write_data(raw_array)
end_time = int(time.time())
if end_time - start_time > 10:
pass
else:
time.sleep(10 - (end_time-start_time))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment