LSTM convergence optimization

LSTM_experiment_epochs_loss.txt

Loss at epoch 0 =  tensor(0.6932, grad_fn=<NllLossBackward>)
Loss at epoch 1000 =  tensor(0.5234, grad_fn=<NllLossBackward>)
Loss at epoch 2000 =  tensor(0.5206, grad_fn=<NllLossBackward>)
Loss at epoch 3000 =  tensor(0.3477, grad_fn=<NllLossBackward>)
Loss at epoch 4000 =  tensor(0.3470, grad_fn=<NllLossBackward>)
Loss at epoch 5000 =  tensor(0.3468, grad_fn=<NllLossBackward>)
Loss at epoch 6000 =  tensor(0.3467, grad_fn=<NllLossBackward>)
Loss at epoch 7000 =  tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 8000 =  tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 9000 =  tensor(0.3466, grad_fn=<NllLossBackward>)
tensor([[[-6.9315e-01, -6.9315e-01],
         [-6.9315e-01, -6.9315e-01]],

        [[-5.3881e-05, -1.0168e+01],
         [-9.8277e+00, -3.8385e-05]],

        [[ 0.0000e+00, -1.8239e-05],
         [-2.3094e+01, -1.0910e+01]],

        [[-3.5763e-06,  0.0000e+00],
         [-1.2558e+01, -2.6699e+01]]], grad_fn=<LogSoftmaxBackward>) tensor([[0, 0],
        [0, 1],
        [1, 1],
        [0, 0]])

LSTM_experiment_source.py

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import numpy as np
#from fastai.learner import *

torch.manual_seed(1)
#torch.cuda.set_device(0)

bs = 2

x_trn = torch.tensor([[1.0000, 1.0000],
        [1.0000, 0.9870],
        [0.9962, 0.9848],
        [1.0000, 1.0000]])#.cuda()

y_trn = torch.tensor([[0, 0],
        [0, 1],
        [1, 1],
        [0, 0]])#.cuda()

n_hidden = 5
n_classes = 2

class TESS_LSTM(nn.Module):
    def __init__(self, nl):
        super().__init__()
        self.nl = nl
        self.rnn = nn.LSTM(1, n_hidden, nl)
        self.l_out = nn.Linear(n_hidden, n_classes)
        self.init_hidden(bs)

    def forward(self, input):
        outp,h = self.rnn(input.view(len(input), bs, -1), self.h)
        return F.log_softmax(self.l_out(outp),dim=1)

    def init_hidden(self, bs):
        self.h = (Variable(torch.zeros(self.nl, bs, n_hidden)),Variable(torch.zeros(self.nl, bs, n_hidden)))

model = TESS_LSTM(1)#.cuda()

loss_function = nn.CrossEntropyLoss()

optimizer = optim.Adam(model.parameters(), lr=0.01)

for epoch in range(10000):  
    model.zero_grad()
    tag_scores = model(x_trn)
    loss = loss_function(tag_scores.reshape(4*bs,n_classes), y_trn.reshape(4*bs))
    loss.backward()
    optimizer.step()

    if epoch%1000==0:
        print("Loss at epoch %d = " %epoch, loss)

print(model(x_trn), y_trn)

Original Post