Recent Advances in the Econometrics of Optimal Policy Design (Part 1)

slides-01.jl

using Plots
using Random
using Statistics

Random.seed!(1)

function eval_pol(pol, n, ε, n_rep)
    ε < 0 && error("ε must be non-negative")
    p_sel = [1, 0, -1]
    R = []

    for r ∈ 1:n_rep
        Z = []
        A = []

        for i ∈ 1:n
            πᵢ = pol(Z, A, n)
            prepend!(A, πᵢ)
            p = .5 + p_sel[πᵢ] * ε
            Y = rand() < p ? 1 : 0
            prepend!(Z, Y)
        end

        fin_rec = pol(Z, A, n)
        reg = ε * (fin_rec == 2) + 2ε * (fin_rec == 3)
        prepend!(R, reg)
    end
    return R
end

function pol(Z, A, n)
    t = length(Z)
    if t < n 
        return rand(1:3)
    else
        return rand(1:3)
    end
end

nv = (1:10) * 200
rv = []
ε = 0.05

for i ∈ nv
    push!(rv, mean(eval_pol(pol, i, ε, 500)))
end

plot(nv, rv, xlab="n", lab="", c=:red)
hline!([ε], lab="", ls=:dash, c=:black)

slides-01.py

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(1)

def eval_pol(pol, n, eps, n_rep):
    if eps < 0:
        raise ValueError("Epsilon must be positive")
    p_sel = [1, 0, -1]
    R = []

    for r in range(n_rep):
        A = []
        Z = []

        for i in range(n):
            pi_i = pol(Z, A, n)
            A = [pi_i] + A
            p = .5 + p_sel[pi_i] * eps
            Y = np.random.binomial(1, p)
            Z = [Y] + Z

        fin_rec = pol(Z, A, n)
        reg = eps * (fin_rec == 1) + 2 * eps * (fin_rec == 2)
        R = [reg] + R   

    return R

def pol(Z, A, n):
    t = len(Z)
    if t < n:
        return np.random.choice([0, 1, 2])
    else:
        return np.random.choice([0, 1, 2])
    
nv = np.arange(1, 11) * 200
rv = []
eps = 0.05

for i in nv:
    rv.append(np.mean(eval_pol(pol, i, eps, 500)))

fig, ax = plt.subplots(figsize=(6, 4))
ax.plot(nv, rv, c="r")
ax.axhline(y=eps, c="k", linestyle="--")

slides-02.jl

using Plots
using Random
using Statistics

Random.seed!(1)

function eval_pol(pol, n, δ, n_rep; bern=true)
    (0 < δ < 1) || error("δ must be in (0, 1)")
    
    regret_matrix = fill(NaN, n_rep, n)

    for i ∈ 1:n_rep
        Z = []
        A = []

        for t ∈ 1:n
            Yₜ = bern ? [rand() < .5, rand() < .5δ] : [rand(), δ * rand()]
            asₜ = pol(Z, A)
            regret_matrix[i, t] = (asₜ == 2) * (.5 - .5δ)
            push!(Z, Yₜ)
            push!(A, asₜ)
        end

        regret_matrix[i, :] .= cumsum(regret_matrix[i, :])
    end

    return regret_matrix
end

function unif_alloc(Z, A)
    return ((length(Z) + 1) % 2) + 1
end

R1_unif = eval_pol(unif_alloc, 10_000, 5/6, 10)
R2_unif = eval_pol(unif_alloc, 10_000, 5/6, 10, bern=false)

p1 = plot(vec(mean(R1_unif, dims=1)), lab="", xlab="t", ylab="Ave. acc. regret: Bernoulli dist.")
p2 = plot(vec(mean(R2_unif, dims=1)), lab="", xlab="t", ylab="Ave. acc. regret: uniform dist.")

plot(p1, p2)

slides-02.py

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(1)

def eval_pol(pol, n, delta, n_rep, bern=True):
    if not (0 < delta < 1):
        raise ValueError("delta must be in (0, 1)")
    
    regret_matrix = np.empty((n_rep, n))

    for i in range(n_rep):
        Z = []
        A = []

        for t in range(n):
            if bern:
                Y_t = [np.random.binomial(1, .5), np.random.binomial(1, delta/2)]
            else:
                Y_t = [np.random.uniform(0, 1), np.random.uniform(0, delta)]

            as_t = pol(Z, A)
            regret_matrix[i, t] = (as_t == 2) * (0.5 - delta/2)
            Z.append(Y_t[as_t - 1])
            A.append(as_t)

        regret_matrix[i] = np.cumsum(regret_matrix[i])
        
    return regret_matrix

def unif_alloc(Z, A):
    return ((len(Z) + 1) % 2) + 1

R1_unif = eval_pol(unif_alloc, 10_000, 5/6, 10)
R2_unif = eval_pol(unif_alloc, 10_000, 5/6, 10, bern=False)

fig, axs = plt.subplots(1, 2, figsize=(8, 4))
axs[0].plot(R1_unif.mean(axis=0), c="b")
axs[0].set(xlabel="t", ylabel="Ave. acc. regret: Bernoulli dist.")
axs[1].plot(R2_unif.mean(axis=0), c="b")
axs[1].set(xlabel="t", ylabel="Ave. acc. regret: uniform dist.")