-
-
Save jcrubino/acb6299bc3d1a08858a3 to your computer and use it in GitHub Desktop.
coinmarketcaps
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ''' | |
| notes | |
| ''' | |
| import json | |
| import urllib | |
| from collections import defaultdict, OrderedDict | |
| import time | |
| import threading | |
| from Queue import Queue | |
| from random import random | |
| clean_price = lambda price: ''.join([x for x in price if x not in ' ,%?!@#$%^&*()']) | |
| # fetches price data over a given period | |
| def fetch_price_data(asset_name, period): | |
| ''' | |
| fetches price data over a given period of time in periods | |
| periods: '1d', '7d', '30d', '90d', '180d', '365d','all' | |
| asset_name: any listed on coinmarket cap | |
| ''' | |
| periods = ['1d', '7d', '30d', '90d', '180d', '365d','all'] | |
| if period not in periods: | |
| raise(Exception("given period not valid")) | |
| url = 'http://coinmarketcap.com/static/generated_pages/currencies/datapoints/{0}-{1}.json'.format(asset_name, period) | |
| return json.loads(urllib.urlopen(url).read()) | |
| class PriceDataWorker(threading.Thread): | |
| ''' | |
| threaded worker to fetch price point data from coinmarketcap | |
| ''' | |
| def __init__(self, work_q, results_q): | |
| threading.Thread.__init__(self) | |
| self.work = work_q | |
| self.results = results_q | |
| def run(self): | |
| while True: | |
| task = self.work.get() | |
| if task == None: | |
| self.work.task_done() | |
| break | |
| result = task() | |
| self.work.task_done() | |
| if 'message' in result.keys(): | |
| print(result) | |
| print(task.pack) | |
| self.results.put(result) | |
| class Task(object): | |
| ''' | |
| task that retrives data | |
| params: | |
| task - a three element list like data structure with: | |
| name - private_name returned from coinmarketcap | |
| period - '1d', '7d', '30d', '90d', '180d', '365d','all' | |
| symbol - exchange traded symbol | |
| will pass if data not available | |
| will sleep up to 10s if network funky then pass on the task | |
| ''' | |
| def __init__(self, task): | |
| self.name = task[0] | |
| self.period = task[1] | |
| self.symbol = task[2] | |
| self.pack = self.name, self.period, self.symbol | |
| def __call__(self): | |
| done = 0 | |
| sleep = 1 | |
| name, period, symbol = self.pack | |
| while done == 0: | |
| try: | |
| data = fetch_price_data(name, period) | |
| data['symbol'] = symbol | |
| data['period'] = period | |
| data['private_name'] = name | |
| return data | |
| except Exception as e: | |
| if str(e) == 'No JSON object could be decoded': | |
| done = 1 | |
| err = {} | |
| err['status'] = 'FAIL' | |
| err['message'] = str(e)+' Probably Newly Released' | |
| time.sleep(sleep) | |
| sleep *= 1 + random() | |
| if sleep > 10: | |
| err = {} | |
| err['status'] = 'FAIL' | |
| err['message'] = str(e) | |
| return err | |
| def __str__(self): | |
| return ' '.join(self.pack) | |
| def __repr__(self): | |
| return 'Task: '+' '.join(self.pack) | |
| class PriceDataManager(object): | |
| def __init__(self, items, threads=8, period='365d'): | |
| self.input = items | |
| self.work_q = Queue() | |
| self.results_q = Queue() | |
| self.end = None | |
| self.threads = threads | |
| if self.threads > len(items): | |
| self.threads = len(items) | |
| self.results = [] | |
| def start(self): | |
| try: | |
| start = time.time() | |
| for data in self.input: | |
| self.work_q.put(Task(data)) | |
| for n in range(self.threads): | |
| self.work_q.put(self.end) | |
| for n in range(self.threads): | |
| if n % 1 == 0: | |
| time.sleep(2) | |
| t = PriceDataWorker(self.work_q, self.results_q) | |
| t.setDaemon(True) | |
| t.start() | |
| while self.work_q.unfinished_tasks > 0: | |
| time.sleep(2) | |
| print("Tasks Left: {0}".format(self.work_q.unfinished_tasks)) | |
| self.time = time.time() - start | |
| for n in xrange(len(self.input)): | |
| self.results.append(self.results_q.get()) | |
| except KeyboardInterrupt: | |
| for n in range(self.threads): | |
| self.work_q.put(self.end) | |
| try: | |
| while self.results_q.unfinished_tasks > 0: | |
| self.results.append(self.results_q.get()) | |
| except Empty: | |
| pass | |
| class Coinmarketcaps(object): | |
| ''' | |
| Coinmarketcap Data http client | |
| Data from Coinmarketcap.com using Kimono public api. | |
| Coinmarketcap Kimono api updates every 15 minutes. | |
| Pulls data from Kimono api on init | |
| e.g. | |
| >>> cm = Coinmarketcaps() | |
| >>> cm.book[0] | |
| {'name':'Bitcoin, 'symbol':'BTC, ...} | |
| >>> cm.price_data(100, 8) # fetches top 100 marketcap weighted coins with 8 threads | |
| # wait a spell (not too long) | |
| # lots of data returned, marketcaps in btc & usd, daily price points if available up to 365days | |
| ''' | |
| def __init__(self): | |
| self.raw_results = None | |
| self._last_update = None | |
| self.book = defaultdict(dict) | |
| self._api_ = defaultdict(dict) | |
| self._odds_ = [] | |
| self.__totals__ = {} | |
| self.update() | |
| self.sort_by_market_cap(None) | |
| def update(self): | |
| self.raw_results = json.load(urllib.urlopen("https://www.kimonolabs.com/api/8vtzwy3i?apikey=NyedFY6zXC2UQTt99yWWWdFcLxAeC3m1")) | |
| self._last_update = time.time() | |
| for value in self.raw_results: | |
| if type(value) == dict: | |
| keys = value.keys() | |
| if 'api' in keys and 'symbol' in keys: | |
| api = value['api'] | |
| self._api_[api].update(value) | |
| if 'symbol' not in keys: | |
| self._odds_.append(value) | |
| def time_since_last_update(self): | |
| if self._last_update == None: | |
| return None | |
| else: | |
| return time.time() - self._last_update | |
| def process_results(self, lim=None, sort='market_cap'): | |
| ''' | |
| params | |
| lim: | |
| is the number of results to include ranked by sort | |
| None means include all results | |
| 100 means include top 100 results | |
| sort: | |
| the criteria to rank results by. | |
| trade_volume, market_cap, last_price | |
| processes raw results, returns dict with name of coin and average or available: | |
| last_price, | |
| market_cap, | |
| trade_volume | |
| mcap_weight # market cap weight for asset | |
| pcap_weight # price weighted ratio for asset | |
| tvol_weight # trade volume ratio for asset | |
| ''' | |
| if sort not in ['trade_volume', 'market_cap', 'last_price']: | |
| raise(Exception("Results not sortable")) | |
| # redundancy needed if sorts run more than once | |
| self.book = defaultdict(dict) | |
| mcap_total = 0 | |
| tvol_total = 0 | |
| price_total= 0 | |
| for item in self.raw_results['results']['coinmarketcaps']: | |
| keys = item.keys() | |
| name = item['name']['text'].lower() | |
| if 'symbol' in keys: | |
| symbol = item['symbol'].upper() | |
| if 'latest_price' in keys: | |
| last_price = item['latest_price']['text'] | |
| if 'market_cap' in keys: | |
| market_cap = item['market_cap'] | |
| if 'trade_volume' in keys: | |
| trade_volume = item['trade_volume']['text'] | |
| markets_url = item['trade_volume']['href'] | |
| try: | |
| last_price = float( filter(lambda x: x.isdigit() or x == '.', last_price)) | |
| market_cap = float( filter(lambda x: x.isdigit() or x == '.', market_cap)) | |
| trade_volume = float( filter(lambda x: x.isdigit() or x == '.', trade_volume)) | |
| except ValueError: | |
| continue | |
| self.book[symbol]['last_price'] = last_price | |
| self.book[symbol]['market_cap'] = market_cap | |
| self.book[symbol]['trade_volume'] = trade_volume | |
| self.book[symbol]['markets_url'] = markets_url | |
| self.book[symbol]['private_name'] = markets_url.split('/')[-2] | |
| self.book[symbol]['symbol'] = symbol | |
| # where sorting takes place | |
| items = self.book.items() | |
| items.sort(key=lambda x: x[1][sort]) | |
| items.reverse() | |
| items = items[0:lim] | |
| keys = [x[0] for x in items] | |
| # prunes final results, just rerun to recalculate from beginning | |
| for symbol in self.book.keys(): | |
| if symbol not in keys: | |
| del self.book[symbol] | |
| for key, book in items: | |
| for k,v in book.items(): | |
| if k == 'market_cap': | |
| mcap_total += v | |
| if k == 'last_price': | |
| price_total += v | |
| if k == 'trade_volume': | |
| tvol_total += v | |
| self.__totals__['marketcap'] = mcap_total | |
| self.__totals__['total_price'] = price_total | |
| self.__totals__['volume_total'] = tvol_total | |
| for key, book in self.book.items(): | |
| mcap = book['market_cap'] | |
| self.book[key]['marketcap_ratio'] = mcap / mcap_total | |
| pcap = book['last_price'] | |
| self.book[key]['price_ratio'] = pcap / price_total | |
| tvol = book['trade_volume'] | |
| self.book[key]['volume_ratio'] = tvol / tvol_total | |
| def __sort__(self, lim, sort): | |
| if sort not in ['trade_volume', 'market_cap', 'last_price']: | |
| raise(Exception("Results not sortable")) | |
| self.process_results(lim, sort) | |
| book = OrderedDict() | |
| items = self.book.items() | |
| items.sort(key=lambda x:x[1]['market_cap']) | |
| items.reverse() | |
| book.update(items) | |
| self.book = book | |
| def sort_by_market_cap(self, lim): | |
| self.__sort__(lim, 'market_cap') | |
| def sort_by_volume(self, lim): | |
| self.__sort__(lim, 'trade_volume') | |
| def sort_by_price(self, lim): | |
| self.__sort__(lim, 'last_price') | |
| def get_items(self, n): | |
| return [(x['private_name'], '365d', x['symbol']) for x in self.book.values()[0:n]] | |
| def get_price_data(self, n=100, threads=8): | |
| ''' | |
| multithreaded price point method | |
| ctrl-c safe | |
| results still possible if method shutdown early | |
| nan results not returned | |
| ''' | |
| thread_manager = PriceDataManager(self.get_items(n), threads) | |
| try: | |
| thread_manager.start() | |
| self.price_data = thread_manager.results | |
| except Exception as e: | |
| self.price_data = thread_manager.results | |
| return self.price_data |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment