astoeffelbauer

class QLearningAgent(acme.Actor):
    
    def __init__(self, env_specs=None, step_size=0.1):
        
        # Black Jack dimensions
        self.Q = np.zeros((32,11,2,2))
        
        # set step size
        self.step_size = step_size
        

astoeffelbauer

class SarsaAgent(acme.Actor):
    
    def __init__(self, env_specs=None, epsilon=0.1, step_size=0.1):
        
        # in Black Jack, we have the following dimensions
        self.Q = np.zeros((32,11,2,2))
        
        # epsilon for policy and step_size for TD learning
        self.epsilon = epsilon
        self.step_size = step_size

astoeffelbauer

# first initialize env and agent
# env = ...
agent = RandomAgent()

# repeat for a number of episodes
for episode in range(10):

    # make first observation
    timestep = env.reset()
    agent.observe_first(timestep)

astoeffelbauer

class RandomAgent(acme.Actor):
    """A random agent for the Black Jack environment."""
    
    def __init__(self):
        
        # init action values, will not be updated by random agent
        self.Q = np.zeros((32,11,2,2))
        
        # specify the behavior policy
        self.behavior_policy = lambda q_values: np.random.choice(2)

astoeffelbauer

# PyTorch nn.Sequential
model = nn.Sequential(
    nn.Embedding(num_embeddings=20000, embedding_dim=50),
    nn.AvgPool1d(kernel_size=50),
    nn.Flatten(start_dim=1),
    nn.Linear(in_features=42, out_features=1),
    nn.Sigmoid()
)

astoeffelbauer

# PyTorch nn.Module Subclassing
class Model(nn.Module):
    
    def __init__(self):
        super(Model, self).__init__()
        self.embedding_layer = nn.Embedding(num_embeddings=20000,
                                            embedding_dim=50)
        self.pooling_layer = nn.AvgPool1d(kernel_size=50)
        self.fc_layer = nn.Linear(in_features=42, out_features=1)
        

astoeffelbauer

inputs = tf.keras.layers.Input(shape=(42,))
x = tf.keras.layers.Embedding(input_dim=20000,
                              output_dimension=50,
                              input_length=42,
                              mask_zero=True)(inputs)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dense(128, activation='relu')(x)
outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)

model = tf.keras.models.Model(inputs=inputs, outputs=outputs)

astoeffelbauer

class Subclass_Model(tf.keras.Model):
    
    def __init__(self):
        super(Subclass_Model, self).__init__()
        self.embedding_layer = tf.keras.layers.Embedding(input_dim=20000,
                                                         output_dimension=50,
                                                         input_length=42,
                                                         mask_zero=True)
        self.flatten_layer = tf.keras.layers.Flatten()
        self.fc1_layer =  tf.keras.layers.Dense(128, activation='relu')

astoeffelbauer

model = tf.keras.Sequential([
    tf.keras.layers.Embedding(input_dim=20000,
                              output_dimension=50,
                              input_length=42,
                              mask_zero=True),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dense(1, activation='sigmoid')])

astoeffelbauer

#define the loss fn and optimizer
criterion = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

#initialize empty list to track batch losses
batch_losses = []

#train the neural network for 5 epochs
for epoch in range(5):