check front-runners for VIX movement

vix_sensitivities.py

# %matplotlib inline
import numpy as np
import pandas as pd
import pandas.io as io
import talib as ta
import datetime as dt
import math
from minepy import MINE
from pandas.io.data import DataReader
import urllib
from joblib import Memory
import matplotlib as mpl
import matplotlib.pyplot as plt
#
memory = Memory('/tmp/')

@memory.cache()
def fetch_yahoo(ticker):
    df = DataReader(ticker, 'yahoo')
    return df

def read_yahoo_data(tickers):
    lis = list()
    for ticker in tickers:
        df = fetch_yahoo(ticker)
        df['VltyDiff'] = np.nan
        adjustment = df['Adj Close']/df['Close']
        df['Open'] *= adjustment
        df['High'] *= adjustment
        df['Low'] *= adjustment
        df['Close'] = df['Adj Close']
        df['Turnover'] = df['Volume']*df['Close']
        df['lnTurnover'] = df['Turnover'].apply(np.log)
        del df['Adj Close']
        for c in df.columns:
            df = df.rename(columns={c: ticker.replace('^', '').lower()+'_'+c.replace(' ', '').lower()})
        lis.append(df)
    df_aggregated = pd.concat(lis, axis=1)
    return df_aggregated

@memory.cache
def fetch_cboe(filename, columns, skips):
    urlStr = 'http://www.cboe.com/publish/ScheduledTask/MktData/datahouse/{0}.csv'.format(filename)
    try:
        data = urllib.request.urlopen(urlStr)
    except Exception as e:
        s = "Failed to download:\n{0}".format(e);
        print(s)        
    if filename in ['vxstcurrent']:
        df = pd.read_csv(data, header=skips, index_col=0, names=columns, encoding="utf-8")
        df.index = df.index.map(lambda s: dt.datetime.strptime(s.replace('*',''), '%m/%d/%Y'))
    else:
        df = pd.read_csv(data, header=skips, index_col=0, names=columns, parse_dates=True, dayfirst=False, encoding="utf-8")
    return df

def read_cboe_data(tickers, vix_close):
    lis = list() 
    for ticker in tickers:      
        filename, columns, skips = COBE_PRODUCTS[ticker]
        df = fetch_cboe(filename, columns, skips).replace('n/a', np.nan)
        df['VltyDiff'] = df['Close']/vix_close if 'Close' in df.columns else np.nan
        df['Open'] = df['Open'] if 'Open' in df.columns else np.nan
        df['High'] = df['High'] if 'High' in df.columns else np.nan
        df['Low'] = df['Low'] if 'Low' in df.columns else np.nan
        df['Close'] = df['Close'] if 'Close' in df.columns else df['P/C Ratio']
        df['Turnover'] = df['TOTAL'] if 'TOTAL' in df.columns else np.nan
        df['lnTurnover'] = df['TOTAL'].apply(np.log) if 'TOTAL' in df.columns else np.nan
        for c in df.columns:
            df = df.rename(columns={c: ticker.replace('^', '').lower()+'_'+c.replace(' ', '').lower()})
        lis.append(df)
    df_aggregated = pd.concat(lis, axis=1)
    return df_aggregated
 
 # https://github.com/jasonstrimpel/volatility-trading/tree/master/volatility/models
def YangZhang(prices, window=30):
    log_ho = (prices['High'] / prices['Open']).apply(np.log)
    log_lo = (prices['Low'] / prices['Open']).apply(np.log)
    log_co = (prices['Close'] / prices['Open']).apply(np.log)
    log_oc = (prices['Open'] / prices['Close'].shift(1)).apply(np.log)
    log_oc_sq = log_oc**2
    log_cc = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
    log_cc_sq = log_cc**2
    rs = log_ho * (log_ho - log_co) + log_lo * (log_lo - log_co)
    close_vol = log_cc_sq.rolling(window=window,center=False).sum() * (1.0 / (window - 1.0))
    open_vol = log_oc_sq.rolling(window=window,center=False).sum() * (1.0 / (window - 1.0))
    window_rs = rs.rolling(window=window,center=False).sum() * (1.0 / (window - 1.0))
    result = (open_vol + 0.164333 * close_vol + 0.835667 * window_rs).apply(np.sqrt) * math.sqrt(252)
    result[:window-1] = np.nan
    return result
    
def sensitivities_by_ticker(df_original, tickers, funcs, term, target):
    num_tickers = len(tickers)
    num_charts = len(funcs)
    for j, ticker in enumerate([t.replace('^', '').lower() for t in tickers]):
        print(ticker)
        df = df_original.copy()
        df['Open'] = df[ticker+'_open']
        df['High'] = df[ticker+'_high']
        df['Low'] = df[ticker+'_low']
        df['Close'] = df[ticker+'_close']
        df['Turnover'] = df[ticker+'_turnover']
        df['lnTurnover'] = df[ticker+'_lnturnover']
        df['VltyDiff'] = df[ticker+'_vltydiff']
        fig = plt.figure(figsize=(28, 10))
        fig.suptitle('{0} - {1}'.format(ticker, target), fontsize=18)
        fig.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.3, hspace=0.7)
        for k, timing in enumerate(['open', 'high', 'low', 'close']):   
            base_prices = df_original[target+'_close']
            target_prices = df_original[target+'_'+timing]
            if timing == 'high':
                ohlc_func = np.max
            elif timing == 'low':
                ohlc_func = np.max
            else:
                ohlc_func = np.mean
            mine = MINE(alpha=0.6, c=15, est="mic_approx")
            ax = fig.add_subplot(4, num_charts+1, k*(num_charts+1)+1)
            ax.title.set_text('{0} {1}'.format(ticker, timing))
            ax.grid()
            ax.plot(df['Close'])
            bx = ax.twinx()
            bx.plot(target_prices, color='red')
            for i, (f, title) in zip(range(num_charts), funcs):
                x, y = f(df), (target_prices.shift(-term).rolling(term).apply(ohlc_func)/base_prices).apply(np.log) #target_prices.pct_change(-term)
                #print(ticker, timing, title)
                x.name = 0
                y.name = 1
                dc = pd.concat([x, y], axis=1).dropna()
                corr = dc[0].corr(dc[1]) 
                mine.compute_score(dc[0].values, dc[1].values)
                mic = mine.mic()
                cx = fig.add_subplot(4, num_charts+1, k*(num_charts+1)+i+2)
                cx.grid()
                cx.title.set_text('{0}\n{1:.2f}, {2:.2f}'.format(title, corr, mic))
                cx.scatter(x, y, s=3, alpha=0.3)
        fig.savefig('./{0}_{1}.png'.format(target, ticker))
        
TICKER_LIST = ['^VIX', '^GSPC', 'VXX', 'XIV', 'XLF', 'TLT', 'GLD', 'EFA', 'EEM', 'JNK', 'SH', 'DX-Y.NYB',
               #'^OVX', '^GZV', '^VVIX', '^VXGS', '^TYVIX', '^VXEFA', '^VXEEM'
              ]

COBE_PRODUCTS = {
    'TOTAL_PC': ('totalpc', ['CALL', 'PUT', 'TOTAL', 'P/C Ratio'], 2),
    'INDEX_PC': ('indexpc', ['CALL', 'PUT', 'TOTAL', 'P/C Ratio'], 2),
    'EQUITY_PC': ('equitypc', ['CALL', 'PUT', 'TOTAL', 'P/C Ratio'], 2),
    'ETP_PC': ('etppc', ['CALL', 'PUT', 'TOTAL', 'P/C Ratio'], 2),
    'VIX_PC': ('vixpc', ['P/C Ratio', 'PUT', 'CALL', 'TOTAL'], 1),
    #
    'VVIX': ('VVIXtimeseries', ['Close'], 1),
    'VXTH': ('VXTHTimeSeries', ['Close'], 1),
    #'VXN': ('vxncurrent', ['Open', 'High', 'Low', 'Close'], 2),
    'VXST': ('vxstcurrent', ['Open', 'High', 'Low', 'Close'], 3),
    #'SKEW': ('Skewdailyprices', ['Close'], 1),
    'OVX': ('ovxhistory', ['Close'], 1),
    'GVZ': ('gvzhistory', ['Close'], 1),    
    'SRVX': ('srvixclose', ['Close'], 1),
    #
    'CHINA_ETF_VI': ('VXFXIDailyPrices', ['Open', 'High', 'Low', 'Close'], 1),
    'VXGS': ('VXGSDailyPrices', ['Open', 'High', 'Low', 'Close'], 1),
    #'RVX': ('RVXDailyPrices', ['Open', 'High', 'Low', 'Close'], 2),
    'VXV': ('vxvdailyprices', ['Open', 'High', 'Low', 'Close'], 2),
    'TYVIX': ('tyvixdailyprices', ['Open', 'High', 'Low', 'Close'], 2),
    'VXEEM': ('VXEEMDailyPrices', ['Open', 'High', 'Low', 'Close'], 2),
    #'TYX': '',
    #'VXMT': '',    
}

df_yahoo = read_yahoo_data(TICKER_LIST)
df_cboe = pd.concat([df_yahoo, read_cboe_data(COBE_PRODUCTS.keys(), df_yahoo['vix_close'])], axis=1)
target = 'vix' 
term = 5
funcs = [
        (lambda df: ((df['Close']/df['Open']).apply(np.log)),'intd chg'),
        (lambda df: ((df['Close']-df['Low'].rolling(30).quantile(0.9))/(df['High'].rolling(30).quantile(0.1)-df['Low'].rolling(30).quantile(0.9))).apply(np.log),'30d pos'),
        (lambda df: ((df['Close'].pct_change(1)+1).apply(np.log)),'1d chg'),
        (lambda df: ((df['Close'].pct_change(2)+1).apply(np.log)),'2d chg'),
        (lambda df: YangZhang(df, 2),'2d yzvol'),
        (lambda df: YangZhang(df, 5),'5d yzvol'),
        (lambda df: YangZhang(df, 2)/YangZhang(df, 5),'2d/5d yzvol'),
        (lambda df: YangZhang(df, 3)/YangZhang(df, 10),'3d/10d yzvol'),
        #(lambda df: ((df['Close']/pd.rolling_mean(df['Close'], 5)).apply(np.log)),'5day mul'),
        (lambda df: (df['Close']/df['Close'].rolling(10).mean()).apply(np.log),'10d mul'),
        #(lambda df: ((df['Close']-pd.rolling_mean(df['Close'], 5))/pd.rolling_std(df['Close'], 5)),'5day z-score'),
        (lambda df: (df['Close']-df['Close'].rolling(10).mean())/df['Close'].rolling(10).std(),'10d zs'),
        (lambda df: (df['Close']-df['Close'].rolling(20).mean())/df['Close'].rolling(20).std(),'20d zs'),
        (lambda df: (df['Turnover'].pct_change(5)+1).apply(np.log),'5d tro chg'),
        (lambda df: (df['lnTurnover']-df['lnTurnover'].rolling(20).mean())/df['lnTurnover'].rolling(20).mean(),'20d tro zs'),
        (lambda df: (df['Close'].pct_change(1)/(df['lnTurnover']/df['lnTurnover'].rolling(20).mean())),'pr press'),
        (lambda df: (df['VltyDiff']+1).apply(np.log), 'vlty diff'),
        (lambda df: (df['VltyDiff'].pct_change(5)+1).apply(np.log), '5d vlty spred'),
    ]

sensitivities_by_ticker(df_cboe, TICKER_LIST, funcs, term, target)
sensitivities_by_ticker(df_cboe, COBE_PRODUCTS.keys(), funcs, term, target)