Last active
October 5, 2020 04:22
-
-
Save sanand0/7243974 to your computer and use it in GitHub Desktop.
Pull currency and stock data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env python | |
""" | |
Author: Bastin Robin | |
""" | |
import os | |
import logging | |
from urllib import urlencode | |
import datetime | |
import pandas as pd | |
CURRENCIES = 'AUD BRL CAD CHF CNY EUR GBP HKD ILS INR ISK JPY \ | |
KRW MXN MYR NZD PHP PKR RUB SAR SEK SGD TWD XAG XAU XPT'.split(' ') | |
STOCKS = '^FTSE ^GSPC ^IXIC'.split(' ') | |
def get_start_date(end_date): | |
''' | |
get the start date from data.csv | |
''' | |
if os.path.exists('data.csv'): | |
data = pd.read_csv('data.csv') | |
data['Date'] = pd.to_datetime(data["Date"]) | |
return data['Date'].max().date() | |
else: | |
return end_date - datetime.timedelta(days=90) | |
def get_currency(frames, start_date, end_date): | |
''' | |
get the currency updated details | |
''' | |
base = 'http://www.oanda.com/currency/historical-rates/download?' | |
for currency in CURRENCIES: | |
logging.info(currency) | |
currency_url = base + urlencode({ | |
'quote_currency': currency, | |
'end_date': end_date.strftime('%Y-%m-%d'), | |
'start_date': start_date.strftime('%Y-%m-%d'), | |
'period': 'daily', | |
'display': 'absolute', | |
'rate': '0', | |
'data_range': 'c', | |
'price': 'mid', | |
'view': 'graph', | |
'base_currency_0': 'USD', | |
'download': '.csv' | |
}) | |
data = pd.read_csv(currency_url, | |
skiprows=5, skipfooter=4, names= | |
['Date', 'Rate', 'x1', 'x2', 'x3', 'x4']) | |
data['Name'] = currency | |
frames.append(data[['Date', 'Rate', 'Name']]) | |
def get_stock(frames, start_date, end_date): | |
''' | |
get the stock details | |
''' | |
for stock in STOCKS: | |
logging.info(stock) | |
stock_url = 'http://ichart.finance.yahoo.com/table.csv?' + \ | |
urlencode({ | |
's': stock, | |
'a': start_date.month - 1, | |
'b': start_date.day, | |
'c': start_date.year, | |
'd': end_date.month - 1, | |
'e': end_date.day, | |
'f': end_date.year, | |
'g': 'd', | |
'ignore': '.csv', | |
}) | |
data = pd.read_csv(stock_url)[['Date', 'Close']].rename(columns={\ | |
'Close': 'Rate'}) | |
data['Name'] = stock | |
frames.append(data[['Date', 'Rate', 'Name']]) | |
if __name__ == '__main__': | |
logging.basicConfig(level=logging.INFO) | |
END_DATE = datetime.date.today() | |
START_DATE = get_start_date(END_DATE) | |
DATAFRAME = [] | |
get_currency(DATAFRAME, START_DATE, END_DATE) | |
get_stock(DATAFRAME, START_DATE, END_DATE) | |
pd.concat(DATAFRAME).to_csv('data.csv', index=False) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment