Bengio sampling

bengio-sample.py

story = ‘’
ctx = [0] * ctx_window # start with context full of “special” characters

while True:
    x = torch.tensor([ctx], device=device)

    for layer in model:
        x = layer(x)

    counts = x.exp()
    probs = counts / counts.sum(dim=1, keepdim=True)
    i = torch.multinomial(probs, num_samples=1, replacement=True, generator=g).item()
    s = itos[i]
    story += s
    ctx = ctx[1:] + [i]

    if i == 0:
        break

print(f”Story time: {story}”)