Hello RNN Training

training.py

N_EPOCHS = 5000
LR = 0.005
end_early = False
seq_i = ""

net.train() # Ensure net in training mode
for epoch_i in range(N_EPOCHS):
    # Zero out gradients
    optimizer.zero_grad()
    
    # Get net output, calculate loss, and generate gradients
    output = net(data.X)
    loss = criterion(output, data.y)
    loss.backward() # Generate gradients via autodiff
    
    # Step
    # -----------------------------------
    # Clip params
    for param in net.parameters():
        if param.grad is None:
            continue
        grad_val = torch.clamp(param.grad, -5, 5)
    optimizer.step()
    # -----------------------------------
    
    # Track loss
    losses.append(loss.item())
    
    # Qualitative Eval
    if epoch_i % 10 == 0:
        seq_i = net.generate(data, data.string[0], num_steps=len(data.string))
        
        if seq_i == data.string:
            end_early = True
            
    # Stdout
    # --------------------------------
    stdout_str = f'\rEpoch {epoch_i+1}/{N_EPOCHS} -- Loss: {losses[-1]:0.4f} -- Network out: {seq_i}'
    sys.stdout.write(stdout_str)
    sys.stdout.flush()
    # --------------------------------
    
    if end_early:
        print(f"\nEnding early. Converged in {epoch_i} epochs.")
        break
    
plt.plot(losses)