Model Confidence Set Issue: Minimal Working Example

mwe_mcs.R

################################################################
# MODEL CONFIDENCE SET ISSUE: Minimal Working Example          #
#                                                              #
# Code demonstrating differences between different R functions #
# for calculating Hansen, Lunde and Nason's (2011).            #
#                                                              #
# Version: 0.2                                                 #
# Date: 17 May 2009                                            #
################################################################

require(MCS)
require(rugarch)
require(modelconf)
require(parallel)
require(dplyr)

cl <- makeForkCluster(nnodes = 8)   # Running Linux, so can fork.
data(Loss)        # Loss data from the MCS package
loss_mat <- as.matrix(Loss[, 1:10]) # mcsTest requires a "plain" matrix
model_names <- colnames(loss_mat)

# Parameters for functions...
alpha <- 0.2
B <- 50000        # Make this big, in order to reduce the variability.
block_len <- 33   # Block length calculated as in MCS package.

# Calculate MCS using various packages. Takes a while...
set.seed(123456)
rugarch_pkg <- mcsTest(losses = loss_mat, alpha = alpha, nboot = B,
                       nblock = block_len, boot = "block")
set.seed(123456)
mcs_pkg <- MCSprocedure(Loss = loss_mat, alpha = alpha, B = B,
                        statistic = 'TR', k = block_len, cl = cl)
set.seed(123456)
modelconf_pkg <- estMCS(loss = loss_mat, test = "t.range", B = B,
                        l = block_len)
stopCluster(cl)

# Extract and format results for comparison.
rugarch_results <- cbind(Model = model_names[c(rugarch_pkg$excludedR,
                                               rugarch_pkg$includedR)],
                         pval = rugarch_pkg$pvalsR)

mcs_results <- mcs_pkg@show %>%
  as.data.frame %>%
  cbind(Model = row.names(mcs_pkg@show), .) %>%
  select(Model, MCS_R) %>%
  arrange(MCS_R)

modelconf_results <- modelconf_pkg %>%
  as.data.frame %>%
  cbind(Model = row.names(modelconf_pkg), .) %>%
  select(Model, `MCS p-val`) %>%
  arrange(`MCS p-val`)

# Inspect results
#   rugarch: Only sGARCH-ghyp & sGARCH-sged in SSM
print(tail(rugarch_results, 5))
# Model                   
# [6,] "sGARCH-sstd"  "0.06714"
# [7,] "sGARCH-snorm" "0.06714"
# [8,] "sGARCH-jsu"   "0.06714"
# [9,] "sGARCH-ghyp"  "0.33244"
# [10,] "sGARCH-sged"  "1"    

#   MCS: only returns SSM, which includes 7 models.
print(tail(mcs_results, 5))
# 3  sGARCH-ghyp 0.82144
# 4   sGARCH-ged 0.96256
# 5   eGARCH-std 0.96704
# 6 sGARCH-snorm 0.99584
# 7  sGARCH-sged 1.00000

#   modelconf: SSM includes 7 models.
print(tail(modelconf_results, 5))
# Model MCS p-val
# 6   sGARCH-ghyp   0.70332
# 7    sGARCH-ged   0.87124
# 8  sGARCH-snorm   0.87124
# 9    eGARCH-std   0.87124
# 10  sGARCH-sged   1.00000

FIXED: Called stopCluster() without supplying the cluster object cl.