Procedure to generate a transition matrix from an MDP

transmat.jl

using POMDPs
using POMDPModelTools

function transition_matrix_a_s_sp(mdp::MDP)
    na = n_actions(mdp)
    ns = n_states(mdp)
    mat = zeros(na, ns, ns) # this should be sparse

    for a in actions(mdp)
        ai = actionindex(mdp, a)
        for s in states(mdp)
            si = stateindex(mdp, s)
            if !isterminal(mdp, s) # if terminal, the transition probabilities are all just zero
                td = transition(mdp, s, a)
                for (sp, p) in weighted_iterator(td)
                    if p > 0.0
                        spi = stateindex(mdp, sp)
                        mat[ai, si, spi] = p
                    end
                end
            end
        end
    end
    return mat
end

using POMDPModels

@show m = SimpleGridWorld()

@show transition_matrix_a_s_sp(m)