super rough q-learning for cartpole-v0

cartpole.py

#!/usr/bin/python3

import numpy as np
import gym
from gym import wrappers

import random
from collections import defaultdict

def process(obs):
    return tuple(int(round(10 * x)) for x in obs)

qvals = defaultdict(lambda: [0, 0])
# entries are [action0reward, action1reward]

discount = 0.99

env = gym.make('CartPole-v0')
env = wrappers.Monitor(env, '/tmp/cartpole')
for i in range(10000):
    explore = max(0, 0.5 - i / 10000)
    learn = max(0.1, 0.5 - i / 10000)
    new_obs = process(env.reset())
    for t in range(1000):
        obs = new_obs
        if random.random() < explore:
            action = random.randint(0, 1)
        else:
            action = np.argmax(qvals[obs])
        new_obs, reward, done, info = env.step(action)
        new_obs = process(new_obs)
        qvals[obs][action] = (1 - learn) * qvals[obs][action] + learn * (reward + discount * max(qvals[new_obs]))
        if done:
            break