entropy_KL.R

library(ggplot2)
theme_set(theme_minimal())

# Visualizing entropy
entropy <- function(p) -sum(p * log(p))
n <- 1e3
p <- runif(n)
q <- 1 - p
z <- matrix(c(p, q), ncol = 2)
ent <- apply(z, 1, entropy)
qplot(q, ent, geom = "line") + 
  ylab("Entropy") + xlab("Probability") +
  ggtitle("Entropy For Two Events")

# Visualizing KL divergence
kld <- function(p, q) sum(p * log(p / q))
n <- 1e3
p <- c(0.6, 0.4)
q1 <- runif(n)
q2 <- 1 - q1
z <- matrix(c(q1, q2), ncol = 2)
kl <- apply(z, 1, kld, p = p)
qplot(q1, kl, geom = "line") +
  ylab("KL Divergence") + 
  xlab("Probability") +
  geom_vline(xintercept = p[1], colour = "grey")