Script that combines all stock data from different sources together

code.py

import os
import pandas as pd
import numpy as np
from datetime import date
import VIX
import new_earnings
import tiingo_data as tii
import US_bond
import crisis_dataset

script_location = os.getcwd()

#get VIX Volatility index
vix_df = VIX.get_vix()

#get 10Y_bond index
usa_bond = US_bond.download_tables()

#get last crisis date data
last_crisis = crisis_dataset.get_dates()

#Get all companies sector, industry, location, etc
metadata = tii.fetch_metadata()

# Get earnings esteem
dow30  = ["AXP", "AMGN", "AAPL", "BA", "CAT", "CSCO", "CVX", "GS", "HD", "HON", "IBM", "INTC", "JNJ", "KO", "JPM", "MCD", "MMM", "MRK", "MSFT", "NKE", "PG", "TRV", "UNH", "CRM", "VZ", "V", "WBA", "WMT", "DIS", "DOW"]

todays_date = date.today()

#a stock symbol to get data
for symbol in dow30[:]:
    #How long historical data we need
    historical_dates_range = 1450
    #How long Moving Average we take
    MA_gener_days = 180
    
    # Hitorical Prices Open, Close, Low, High, Volume for historical data
    stock_dataset = tii.fetch_stock(symbol, todays_date, historical_dates_range)
    # Adding a Moving Average and price position over/below MA to the previous dataset
    #data = tii.create_dataset(stock_dataset, MA_gener_days)
    # Getting stock market cap, PE Ratio, PB Ratio
    fundamentals  =   tii.fetch_fundamentals(symbol, todays_date, historical_dates_range)
    # Financial KPIs
    statements =   tii.fetch_statements(symbol, todays_date, historical_dates_range)
    # Get stock sector and industry
    stock_metadata = metadata[metadata.ticker == symbol.lower()][["sector","industry"]].copy()
    # stock dividents & earnings
    stock_earnings, stock_dividends = new_earnings.get_earn_and_dividends(symbol)    
    
    # Combining all stock's data
    big_dataset = tii.combine_tables(stock_dataset, statements, fundamentals, stock_metadata, historical_dates_range)
    big_dataset = big_dataset.join(vix, how = 'left')
    big_dataset.dropna(inplace = True)
    
    #Combining data with US Bonds
    big_dataset = big_dataset.join(usa_bond, how = 'left')
    #Combining data with earnings & dividends info
    big_dataset = big_dataset.join(stock_earnings, how = 'left')
    big_dataset = big_dataset.join(stock_dividends, how = 'left')
    
    big_dataset.sort_values(by = 'date', axis = 0, ascending = False, inplace = True)
    big_dataset["10Y_bond_MoM"] = big_dataset["10Y_bonds"]-big_dataset["10Y_bonds"].shift(-22)
    big_dataset["Debt-to-Equity_Ratio"] = big_dataset["totalAssets"]/big_dataset["totalLiabilities"]
    big_dataset["DividendsYield"] = big_dataset["payDiv"]/big_dataset["marketCap"]
    big_dataset["PayoutRatio"] = big_dataset["payDiv"]/big_dataset["grossProfit"]
    big_dataset["Acc_Rec_Pay_Ration"] = big_dataset["acctRec"]/big_dataset["acctPay"]
    big_dataset["Earnings_per_stock"] = big_dataset["epsDil"]/big_dataset["close"]
    big_dataset["future_30dprice_change"] = (big_dataset["close"].shift(22) - big_dataset["close"])/big_dataset["close"].shift(22)
    #big_dataset = big_dataset[["roe", "longTermDebtEquity", "grossMargin", "revenueQoQ", "rps", "epsQoQ", "piotroskiFScore", "currentRatio", "roa", "profitMargin","peRatio", "pbRatio","trailingPEG1Y","VIX_high","sector","industry","10Y_bonds", "10Y_bond_MoM","Debt-to-Equity_Ratio","DividendsYield","PayoutRatio","Acc_Rec_Pay_Ration","Earnings_per_stock","dividends_change","prev_div_change","days_after_divid_report","surprise_%", "expected_growth", "previous_surprise","days_after_earn_report"]]
    
    #TO DO 
    #CALC Debt-to-Equity_Ratio,	DividendsYield,	Payout ratio, Acc_Rec_Pay_Ration, YoY Debt ratio, YoY eps, YoY investments
    #REPLACE DividendsYield,	Payout ratio by estimates in Future Periods
    
    csv_file = symbol + ".csv"
    path_to_all_data = os. path. join(script_location, "CSVs")
    path_to_all_data = os. path. join(path_to_all_data, csv_file)
    big_dataset.to_csv(path_to_all_data)