Skip to content

Instantly share code, notes, and snippets.

@peacing
Created March 20, 2022 21:55
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save peacing/1b82dde14943b6500ccbaa4ce3e8b6ea to your computer and use it in GitHub Desktop.
saving crypto data
import requests
from datetime import datetime
import pandas as pd
import awswrangler as wr
import boto3
session = boto3.session.Session(aws_access_key_id=key_access_aws, aws_secret_access_key=key_secret_aws)
s3 = session.client('s3')
def get_data_from_api():
"""
Get data from API and transform it to DataFrame
"""
api_url = "https://min-api.cryptocompare.com/data/pricemulti?fsyms=BTC,ETH,REP,DASH&tsyms=USD"
r = requests.get(api_url).json()
now = datetime.now()
dtime = now.strftime('%Y-%m-%d %H:%M:%S')
date_key = now.strftime('%Y%m%d')
month_key = now.strftime('%Y%m')
datas = []
for rec in r:
currency = rec
price = r[currency]['USD']
print(rec, price, now)
record = {'currency': currency, 'price': price, 'datetime': dtime, 'date_key': date_key, 'month_key': month_key}
datas.append(record)
return datas
datas = get_data_from_api()
df = pd.DataFrame(datas)
wr.s3.to_parquet(
df=df,
boto3_session=session,
path=f"s3://cryptolake-data/crypto_prices/",
dataset=True
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment