Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
saving crypto data
import requests
from datetime import datetime
import pandas as pd
import awswrangler as wr
import boto3
session = boto3.session.Session(aws_access_key_id=key_access_aws, aws_secret_access_key=key_secret_aws)
s3 = session.client('s3')
def get_data_from_api():
"""
Get data from API and transform it to DataFrame
"""
api_url = "https://min-api.cryptocompare.com/data/pricemulti?fsyms=BTC,ETH,REP,DASH&tsyms=USD"
r = requests.get(api_url).json()
now = datetime.now()
dtime = now.strftime('%Y-%m-%d %H:%M:%S')
date_key = now.strftime('%Y%m%d')
month_key = now.strftime('%Y%m')
datas = []
for rec in r:
currency = rec
price = r[currency]['USD']
print(rec, price, now)
record = {'currency': currency, 'price': price, 'datetime': dtime, 'date_key': date_key, 'month_key': month_key}
datas.append(record)
return datas
datas = get_data_from_api()
df = pd.DataFrame(datas)
wr.s3.to_parquet(
df=df,
boto3_session=session,
path=f"s3://cryptolake-data/crypto_prices/",
dataset=True
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment