multi-armed-bundit simulation with simple e-greedy strategy

ma_bundit.R

# m=arms, n=biddings, h=trials
simulation <- function(m,n,h,binom,pois,strategies) {
  ss <- length(strategies)
  
  score <- list()
  bid <- list()
  for (i in 1:ss) { score[[i]] <- rep(0,n) }
    
  for (t in 1:h) {
    for (i in 1:ss) {
      prev <- rep(0,m)
      if (t>1) {
        prev[bid[[i]]] <- result[bid[[i]]]
      }
      bid[[i]] <- (strategies[[i]])(m,n,h,t,prev)
    }
    result <- rbinom(m,binom[2],binom[1])+rpois(m,pois[2])*pois[1]
    for (i in 1:ss) { 
      score[[i]] <- score[[i]]+result[bid[[i]]] 
    }
  }
  for (i in 1:ss) { score[[i]] <- sum(score[[i]]) }
  return(score)
}

# random for each iteration
strategy0 <- function() {
  function(m,n,h,t,prev) {
    r<-floor(runif(n)*m)+1
    ifelse(r>m,m,r)
  }
}
 
# epsilon greedy
strategy1 <- function(e) {
  df <- data.frame(e=e)
  function(m,n,h,t,prev) {
    best <- 0
    if (t>1) { best <- which.max(prev) }
    
    e <- df$e
    if (best==0) { e <- 1 }
    
    ne <- floor(n*e)
    r <- floor(runif(ne)*(m-1))+1
    r <- ifelse(r>=best,r+1,r)
    r <- ifelse(r>m,m,r)
    c(rep(best,n-ne),r)
  }
}

main <- function(trials=10,m=10,n=30,h=100,binom=c(0.5,10),pois=c(0.01,100)) {
    init_ss <- function() {
    list(
      strategy0(),
      strategy1(0.1),
      strategy1(0.2),
      strategy1(0.3),
      strategy1(0.4),
      strategy1(0.5)
    )
  }
  ss_len <- length(init_ss())
  
  sum <- rep(0,ss_len)
  for (i in 1:trials) {
    ss <- init_ss()
    res <- simulation(m=m,n=n,h=h,binom=binom,pois=pois,strategies=ss)
    for (j in 1:ss_len) {
      sum[j] <- sum[j]+res[[j]]
    }
  }
  
  print(paste("Average score for ",trials," trials:",sep=""))
  for (j in 1:ss_len) {
    print(paste(" Strategy ",j," = ",sum[j]/trials, sep=""))
  }
}