Bengio deep

bengio-deep.py

model = [
    Embedding(device=device, num_embeddings=vocab_size, embedding_dim=embed_dim),
    Flatten(input_dim1=ctx_window, input_dim2=embed_dim),
    Linear(device=device, in_features=ctx_window*embed_dim, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=hidden_size, bias=True),
    Tanh(),
    Linear(device=device, in_features=hidden_size, out_features=vocab_size, bias=True)
]

params = [p for layer in model for p in layer.params()]

# Enable gradients for the learnable parameters
for p in params:
    p.requires_grad = True

# Create RNG
g = torch.Generator(device=device).manual_seed(42)