Bengio basic model

basic-bengio.py

model = [
    Embedding(device=device, num_embeddings=vocab_size, embedding_dim=embed_dim),
    Flatten(device=device, input_dim1=ctx_window, input_dim2=embed_dim),
    Linear(device=device, in_features=ctx_window*embed_dim, out_features=hidden_size, bias=True),
    Tanh(device=device),
    Linear(device=device, in_features=hidden_size, out_features=vocab_size, bias=False)
]

params = [p for layer in model for p in layer.params()]

# Enable gradients for the learnable parameters
for p in params:
    p.requires_grad = True

# Construct train and validation sets (note we leave the data_val
# set as a hold-out test set after we are done tuning hyperparams)
X_trn, Y_trn = build_dataset(data_trn[:20000], ctx_window)
X_val, Y_val = build_dataset(data_trn[20000:22000], ctx_window)
X_trn = X_trn.to(device)
Y_trn = Y_trn.to(device)
X_val = X_val.to(device)
Y_val = Y_val.to(device)

# Create RNG
g = torch.Generator(device=device).manual_seed(42)