Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
LSTM convergence optimization
Loss at epoch 0 = tensor(0.6932, grad_fn=<NllLossBackward>)
Loss at epoch 1000 = tensor(0.5234, grad_fn=<NllLossBackward>)
Loss at epoch 2000 = tensor(0.5206, grad_fn=<NllLossBackward>)
Loss at epoch 3000 = tensor(0.3477, grad_fn=<NllLossBackward>)
Loss at epoch 4000 = tensor(0.3470, grad_fn=<NllLossBackward>)
Loss at epoch 5000 = tensor(0.3468, grad_fn=<NllLossBackward>)
Loss at epoch 6000 = tensor(0.3467, grad_fn=<NllLossBackward>)
Loss at epoch 7000 = tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 8000 = tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 9000 = tensor(0.3466, grad_fn=<NllLossBackward>)
tensor([[[-6.9315e-01, -6.9315e-01],
[-6.9315e-01, -6.9315e-01]],
[[-5.3881e-05, -1.0168e+01],
[-9.8277e+00, -3.8385e-05]],
[[ 0.0000e+00, -1.8239e-05],
[-2.3094e+01, -1.0910e+01]],
[[-3.5763e-06, 0.0000e+00],
[-1.2558e+01, -2.6699e+01]]], grad_fn=<LogSoftmaxBackward>) tensor([[0, 0],
[0, 1],
[1, 1],
[0, 0]])
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import numpy as np
#from fastai.learner import *
torch.manual_seed(1)
#torch.cuda.set_device(0)
bs = 2
x_trn = torch.tensor([[1.0000, 1.0000],
[1.0000, 0.9870],
[0.9962, 0.9848],
[1.0000, 1.0000]])#.cuda()
y_trn = torch.tensor([[0, 0],
[0, 1],
[1, 1],
[0, 0]])#.cuda()
n_hidden = 5
n_classes = 2
class TESS_LSTM(nn.Module):
def __init__(self, nl):
super().__init__()
self.nl = nl
self.rnn = nn.LSTM(1, n_hidden, nl)
self.l_out = nn.Linear(n_hidden, n_classes)
self.init_hidden(bs)
def forward(self, input):
outp,h = self.rnn(input.view(len(input), bs, -1), self.h)
return F.log_softmax(self.l_out(outp),dim=1)
def init_hidden(self, bs):
self.h = (Variable(torch.zeros(self.nl, bs, n_hidden)),Variable(torch.zeros(self.nl, bs, n_hidden)))
model = TESS_LSTM(1)#.cuda()
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
for epoch in range(10000):
model.zero_grad()
tag_scores = model(x_trn)
loss = loss_function(tag_scores.reshape(4*bs,n_classes), y_trn.reshape(4*bs))
loss.backward()
optimizer.step()
if epoch%1000==0:
print("Loss at epoch %d = " %epoch, loss)
print(model(x_trn), y_trn)
@DevMahmoud10

This comment has been minimized.

Copy link
Owner Author

DevMahmoud10 commented Mar 21, 2019

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.