validation of "Twitter mood predicts the stock market"

calm_dji.csv

predict_dji.py

#!/usr/bin/python
import numpy as np
import scipy.sparse
import xgboost as xgb
import pickle
import talib as ta
from datetime import datetime as dt
import pytz

INPUT_LEN = 3
TRAINDATA_DIV = 2

"""
main
"""
rates_fd = open('./calm_dji_2009.csv', 'r')

exchange_dates = []

calm_rates = []
calm_rates_diff = []

exchange_rates = []
exchange_rates_diff = []

prev_calm = 0
prev_exch = 10000

for line in rates_fd:
    splited = line.split(",")
    time = splited[0]
    calm_val = float(splited[1])
    exch_val = float(splited[2])
    exchange_dates.append(time)

    calm_rates.append(calm_val)
    calm_rates_diff.append(calm_val - prev_calm)
    
    exchange_rates.append(exch_val)
    exchange_rates_diff.append(exch_val - prev_exch)
    
    prev_calm = calm_val
    prev_exch = exch_val
    
data_len = len(exchange_rates)
train_len = len(exchange_rates)/TRAINDATA_DIV

print "data size: " + str(data_len)
print "train len: " + str(train_len)

if False:
    bst = xgb.Booster({'nthread':4})
    bst.load_model("./calm.model") 

if True: ### training start
    tr_input_mat = []
    tr_angle_mat = []
    prev_pred = 0
    for ii in xrange(INPUT_LEN, train_len):
        tmp_arr = []
        for jj in xrange(INPUT_LEN):
            tmp_arr.append(exchange_rates_diff[ii-INPUT_LEN+jj])
        for jj in xrange(INPUT_LEN):
            tmp_arr.append(calm_rates_diff[ii-INPUT_LEN+jj])   
        tr_input_mat.append(tmp_arr)

        if exchange_rates_diff[ii + 1] >= 0:
            tr_angle_mat.append(1)
        else:
            tr_angle_mat.append(0)
        
    tr_input_arr = np.array(tr_input_mat)
    tr_angle_arr = np.array(tr_angle_mat)
    dtrain = xgb.DMatrix(tr_input_arr, label=tr_angle_arr)
    param = {'max_depth':6, 'eta':0.2, 'subsumble':0.5, 'silent':1, 'objective':'binary:logistic' }
    watchlist  = [(dtrain,'train')]
    num_round =  3000  #3000 #10 #3000 # 1000
    bst = xgb.train(param, dtrain, num_round, watchlist)

    try_cnt = 0
    correct_cnt = 0
    for ii in xrange(train_len - INPUT_LEN, data_len - 1):
        if exchange_rates[ii + 1] == 0:
            pass
        
        # prediction    
        ts_input_mat = []
        tmp_arr = []
        for jj in xrange(INPUT_LEN):
            tmp_arr.append(exchange_rates_diff[ii-INPUT_LEN+jj])
        for jj in xrange(INPUT_LEN):
            tmp_arr.append(calm_rates_diff[ii-INPUT_LEN+jj])            
        tr_input_mat.append(tmp_arr)
    
        ts_input_arr = np.array(ts_input_mat)
        dtest = xgb.DMatrix(ts_input_arr)
        pred = bst.predict(dtest)
        predicted_prob = pred[0]
    
        if predicted_prob >= 0.5:
            predicted_angle = 1
        else:
            predicted_angle = 0

        try_cnt += 1
        if exchange_rates_diff[ii+1] >= 0 and  predicted_angle == 1:
            correct_cnt += 1
        if exchange_rates_diff[ii+1] < 0 and  predicted_angle == 0:
            correct_cnt += 1            
        
        print(correct_cnt/float(try_cnt))
        
bst.save_model('./calm.model')