Example of discrepancies between SparseVI and VI when handling terminal states

sparse_vi_terminal_states.jl

using POMDPs
using POMDPModelTools
using DiscreteValueIteration 
using Parameters

@with_kw struct TwoStatesMDP <: MDP{Int, Int}
    γ::Float64 = 0.95
end

POMDPs.n_states(mdp::TwoStatesMDP) = 2
POMDPs.states(mdp::TwoStatesMDP) = 1:2
POMDPs.stateindex(mdp::TwoStatesMDP, s) = s 
POMDPs.n_actions(mdp::TwoStatesMDP) = 2
POMDPs.actions(mdp::TwoStatesMDP) = 1:2
POMDPs.actionindex(mdp::TwoStatesMDP, a) = a
POMDPs.discount(mdp::TwoStatesMDP) = mdp.γ

function POMDPs.transition(mdp::TwoStatesMDP, s, a)
    return SparseCat([a], [1.0])
end

function POMDPs.reward(mdp::TwoStatesMDP, s, a, sp)
    return float(sp == 2)
end

POMDPs.isterminal(mdp::TwoStatesMDP, s) = s == 2


mdp = TwoStatesMDP()
solver = ValueIterationSolver(verbose = true)
policy = solve(solver, mdp)

println(policy.qmat)

sparsesolver = SparseValueIterationSolver(verbose=true)
policy = solve(sparsesolver, mdp)

println(policy.qmat)