marcel_vos_rct_maze_mdp.py

import numpy as np
from matplotlib import pyplot as plt


def make_Av(N):
    a = 1 - 0.5 * 0.75
    b = 0.5 * 0.25

    A = np.zeros((2 * N + 2, 2 * N + 2))

    # Start
    A[0, 1] = 1

    for i in range(1, 2*N, 2):
        # forward
        A[i, i - 1] = 1 - a
        A[i, i + 2] = a

        # reverse
        A[i + 1, i - 1] = 1 - b
        A[i + 1, i + 2] = b

    # absorbing state
    A[2*N + 1, 2*N + 1] = 1

    # initial state
    v = np.zeros(2*N + 2)
    v[0] = 1

    return A, v

def compute_E(N):
    A, v = make_Av(N)
    I = np.eye(A.shape[0])

    F = (I - A)[:-1, :-1]
    N = np.linalg.inv(F)
    E = np.sum(N[0, :])
    return E


vals = [
    compute_E(N + 1) / compute_E(N)
    for N in range(1, 100)]
print(vals)

plt.plot(vals)
plt.show()