zsunberg

using POMDPs
import POMDPs: create_state, discount, reward
using POMDPToolbox

type ImageMDP <: MDP{Matrix{Int},Int}
    size::Tuple{Int, Int}
end

create_state(mdp::ImageMDP) = Array(Int, mdp.size[1], mdp.size[2])
discount(::ImageMDP) = 0.9

zsunberg

AutoViz.jl

A package for rendering simple scenes primarily consisting of cars on roadways using Cairo.

Usage

The main function is

render(scene)

zsunberg

const Vec2 = SVector{2,Int}
const StateTypes = Union{Vec2, TerminalState}

@with_kw struct SimpleGridWorld <: MDP{StateTypes, Symbol}
    size::Tuple{Int, Int}           = (10,10)
    rewards::Dict{Vec2, Float64}    = Dict(Vec2(4,3)=>-10.0, Vec2(4,6)=>-5.0, Vec2(9,3)=>10.0, Vec2(8,8)=>3.0)
    terminate_in::Set{Vec2}         = Set((Vec2(4,3), Vec2(4,6), Vec2(9,3), Vec2(8,8)))
    tprob::Float64                  = 0.7
    discount::Float64               = 0.95
end

zsunberg

using POMDPs
using POMDPModelTools
using POMDPSimulators
using POMDPPolicies
using StaticArrays
using Parameters
using Random
using BenchmarkTools
using POMDPModels
using Test

zsunberg

using POMDPs
using POMDPModelTools

function transition_matrix_a_s_sp(mdp::MDP)
    na = n_actions(mdp)
    ns = n_states(mdp)
    mat = zeros(na, ns, ns) # this should be sparse

    for a in actions(mdp)
        ai = actionindex(mdp, a)

zsunberg

using ExtendedKalmanFilters
using Distributions
using DelimitedFiles

# We may also want to look at DynamicalSystems.jl
# The package should accept AbstractArrays wherever possible so people can use StaticArrays

# Model semantics
# x_{t+1} = f(x_t, u_t) + w_t
# y_t = h(x_t) + v_t # should the control be an argument of h?

zsunberg

Here are the two ways that I was referring to about augmenting the state space (these are illustrative rather than efficient or complete implementations):

1. Add a single new terminal state

struct VariableDiscountWrapper1{S, A, F<:Function} <: MDP{Union{S, TerminalState}, A}
    m::MDP{S, A}
    discount::F
end

zsunberg

The only code in the entire package initially is

abstract type CommonEnv end

function reset! end
function step! end
function actions end

(of course there will be extensive documentation, etc.)

zsunberg

{
  "name": "MathOptFormat Model",
  "version": {
    "major": 0,
    "minor": 4
  },
  "variables": [
    {
      "name": "x[1,1]"
    },

zsunberg

using ReinforcementLearningZoo
using ReinforcementLearningBase
using ReinforcementLearningCore: NeuralNetworkApproximator, EpsilonGreedyExplorer, QBasedPolicy, CircularCompactSARTSATrajectory
using ReinforcementLearning
using Flux
using Flux: glorot_uniform, huber_loss
import Random
import BSON

RL = ReinforcementLearningBase